[virtio-dev] [RFC PATCH linux] iommu: Add virtio-iommu driver

Bharat Bhushan
2017-06-16T08:48:00Z

Hi Jean

> -----Original Message-----
> From: virtio-dev at lists.oasis-open.org [mailto:virtio-dev at lists.oasis-
> open.org] On Behalf Of Jean-Philippe Brucker
> Sent: Saturday, April 08, 2017 12:53 AM
> To: iommu at lists.linux-foundation.org; kvm at vger.kernel.org;
> virtualization at lists.linux-foundation.org; virtio-dev at
lists.oasis-open.org
> Cc: cdall at linaro.org; will.deacon at arm.com; robin.murphy at arm.com;
> lorenzo.pieralisi at arm.com; joro at 8bytes.org; mst at redhat.com;
> jasowang at redhat.com; alex.williamson at redhat.com;
> marc.zyngier at arm.com
> Subject: [virtio-dev] [RFC PATCH linux] iommu: Add virtio-iommu driver
>
> The virtio IOMMU is a para-virtualized device, allowing to send IOMMU
> requests such as map/unmap over virtio-mmio transport. This driver should
> illustrate the initial proposal for virtio-iommu, that you hopefully
received
> with it. It handle attach, detach, map and unmap requests.
>
> The bulk of the code is to create requests and send them through virtio.
> Implementing the IOMMU API is fairly straightforward since the virtio-iommu
> MAP/UNMAP interface is almost identical. I threw in a custom
> map_sg() function which takes up some space, but is optional. The core
> function would send a sequence of map requests, waiting for a reply
> between each mapping. This optimization avoids yielding to the host after
> each map, and instead prepares a batch of requests in the virtio ring and
> kicks the host once.
>
> It must be applied on top of the probe deferral work for IOMMU, currently
> under discussion. This allows to dissociate early driver detection and
device
> probing: device-tree or ACPI is parsed early to find which devices are
> translated by the IOMMU, but the IOMMU itself cannot be probed until the
> core virtio module is loaded.
>
> Enabling DEBUG makes it extremely verbose at the moment, but it should be
> calmer in next versions.
>
> Signed-off-by: Jean-Philippe Brucker <jean-philippe.brucker at
arm.com>
> ---
>  drivers/iommu/Kconfig             |  11 +
>  drivers/iommu/Makefile            |   1 +
>  drivers/iommu/virtio-iommu.c      | 980
> ++++++++++++++++++++++++++++++++++++++
>  include/uapi/linux/Kbuild         |   1 +
>  include/uapi/linux/virtio_ids.h   |   1 +
>  include/uapi/linux/virtio_iommu.h | 142 ++++++
>  6 files changed, 1136 insertions(+)
>  create mode 100644 drivers/iommu/virtio-iommu.c  create mode 100644
> include/uapi/linux/virtio_iommu.h
>
> diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig index
> 37e204f3d9be..8cd56ee9a93a 100644
> --- a/drivers/iommu/Kconfig
> +++ b/drivers/iommu/Kconfig
> @@ -359,4 +359,15 @@ config MTK_IOMMU_V1
>
>  	  if unsure, say N here.
>
> +config VIRTIO_IOMMU
> +	tristate "Virtio IOMMU driver"
> +	depends on VIRTIO_MMIO
> +	select IOMMU_API
> +	select INTERVAL_TREE
> +	select ARM_DMA_USE_IOMMU if ARM
> +	help
> +	  Para-virtualised IOMMU driver with virtio.
> +
> +	  Say Y here if you intend to run this kernel as a guest.
> +
>  endif # IOMMU_SUPPORT
> diff --git a/drivers/iommu/Makefile b/drivers/iommu/Makefile index
> 195f7b997d8e..1199d8475802 100644
> --- a/drivers/iommu/Makefile
> +++ b/drivers/iommu/Makefile
> @@ -27,3 +27,4 @@ obj-$(CONFIG_TEGRA_IOMMU_SMMU) += tegra-
> smmu.o
>  obj-$(CONFIG_EXYNOS_IOMMU) += exynos-iommu.o
>  obj-$(CONFIG_FSL_PAMU) += fsl_pamu.o fsl_pamu_domain.o
>  obj-$(CONFIG_S390_IOMMU) += s390-iommu.o
> +obj-$(CONFIG_VIRTIO_IOMMU) += virtio-iommu.o
> diff --git a/drivers/iommu/virtio-iommu.c b/drivers/iommu/virtio-iommu.c
> new file mode 100644 index 000000000000..1cf4f57b7817
> --- /dev/null
> +++ b/drivers/iommu/virtio-iommu.c
> @@ -0,0 +1,980 @@
> +/*
> + * Virtio driver for the paravirtualized IOMMU
> + *
> + * This program is free software; you can redistribute it and/or modify
> + * it under the terms of the GNU General Public License version 2 as
> + * published by the Free Software Foundation.
> + *
> + * This program is distributed in the hope that it will be useful,
> + * but WITHOUT ANY WARRANTY; without even the implied warranty of
> + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
> + * GNU General Public License for more details.
> + *
> + * You should have received a copy of the GNU General Public License
> + * along with this program; if not, write to the Free Software
> + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307,
> USA.
> + *
> + * Copyright (C) 2017 ARM Limited
> + *
> + * Author: Jean-Philippe Brucker <jean-philippe.brucker at arm.com>
*/
> +
> +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
> +
> +#include <linux/amba/bus.h>
> +#include <linux/delay.h>
> +#include <linux/dma-iommu.h>
> +#include <linux/freezer.h>
> +#include <linux/interval_tree.h>
> +#include <linux/iommu.h>
> +#include <linux/module.h>
> +#include <linux/of_iommu.h>
> +#include <linux/of_platform.h>
> +#include <linux/platform_device.h>
> +#include <linux/virtio.h>
> +#include <linux/virtio_config.h>
> +#include <linux/virtio_ids.h>
> +#include <linux/wait.h>
> +
> +#include <uapi/linux/virtio_iommu.h>
> +
> +struct viommu_dev {
> +	struct iommu_device		iommu;
> +	struct device			*dev;
> +	struct virtio_device		*vdev;
> +
> +	struct virtqueue		*vq;
> +	struct list_head		pending_requests;
> +	/* Serialize anything touching the vq and the request list */
> +	spinlock_t			vq_lock;
> +
> +	struct list_head		list;
> +
> +	/* Device configuration */
> +	u64				pgsize_bitmap;
> +	u64				aperture_start;
> +	u64				aperture_end;
> +};
> +
> +struct viommu_mapping {
> +	phys_addr_t			paddr;
> +	struct interval_tree_node	iova;
> +};
> +
> +struct viommu_domain {
> +	struct iommu_domain		domain;
> +	struct viommu_dev		*viommu;
> +	struct mutex			mutex;
> +	u64				id;
> +
> +	spinlock_t			mappings_lock;
> +	struct rb_root			mappings;
> +
> +	/* Number of devices attached to this domain */
> +	unsigned long			attached;
> +};
> +
> +struct viommu_endpoint {
> +	struct viommu_dev		*viommu;
> +	struct viommu_domain		*vdomain;
> +};
> +
> +struct viommu_request {
> +	struct scatterlist		head;
> +	struct scatterlist		tail;
> +
> +	int				written;
> +	struct list_head		list;
> +};
> +
> +/* TODO: use an IDA */
> +static atomic64_t viommu_domain_ids_gen;
> +
> +#define to_viommu_domain(domain) container_of(domain, struct
> +viommu_domain, domain)
> +
> +/* Virtio transport */
> +
> +static int viommu_status_to_errno(u8 status) {
> +	switch (status) {
> +	case VIRTIO_IOMMU_S_OK:
> +		return 0;
> +	case VIRTIO_IOMMU_S_UNSUPP:
> +		return -ENOSYS;
> +	case VIRTIO_IOMMU_S_INVAL:
> +		return -EINVAL;
> +	case VIRTIO_IOMMU_S_RANGE:
> +		return -ERANGE;
> +	case VIRTIO_IOMMU_S_NOENT:
> +		return -ENOENT;
> +	case VIRTIO_IOMMU_S_FAULT:
> +		return -EFAULT;
> +	case VIRTIO_IOMMU_S_IOERR:
> +	case VIRTIO_IOMMU_S_DEVERR:
> +	default:
> +		return -EIO;
> +	}
> +}
> +
> +static int viommu_get_req_size(struct virtio_iommu_req_head *req, size_t
> *head,
> +			       size_t *tail)
> +{
> +	size_t size;
> +	union virtio_iommu_req r;
> +
> +	*tail = sizeof(struct virtio_iommu_req_tail);
> +
> +	switch (req->type) {
> +	case VIRTIO_IOMMU_T_ATTACH:
> +		size = sizeof(r.attach);
> +		break;
> +	case VIRTIO_IOMMU_T_DETACH:
> +		size = sizeof(r.detach);
> +		break;
> +	case VIRTIO_IOMMU_T_MAP:
> +		size = sizeof(r.map);
> +		break;
> +	case VIRTIO_IOMMU_T_UNMAP:
> +		size = sizeof(r.unmap);
> +		break;
> +	default:
> +		return -EINVAL;
> +	}
> +
> +	*head = size - *tail;
> +	return 0;
> +}
> +
> +static int viommu_receive_resp(struct viommu_dev *viommu, int
> +nr_expected) {
> +
> +	unsigned int len;
> +	int nr_received = 0;
> +	struct viommu_request *req, *pending, *next;
> +
> +	pending = list_first_entry_or_null(&viommu->pending_requests,
> +					   struct viommu_request, list);
> +	if (WARN_ON(!pending))
> +		return 0;
> +
> +	while ((req = virtqueue_get_buf(viommu->vq, &len)) != NULL) {
> +		if (req != pending) {
> +			dev_warn(viommu->dev, "discarding stale
> request\n");
> +			continue;
> +		}
> +
> +		pending->written = len;
> +
> +		if (++nr_received == nr_expected) {
> +			list_del(&pending->list);
> +			/*
> +			 * In an ideal world, we'd wake up the waiter for this
> +			 * group of requests here. But everything is painfully
> +			 * synchronous, so waiter is the caller.
> +			 */
> +			break;
> +		}
> +
> +		next = list_next_entry(pending, list);
> +		list_del(&pending->list);
> +
> +		if (WARN_ON(list_empty(&viommu->pending_requests)))
> +			return 0;
> +
> +		pending = next;
> +	}
> +
> +	return nr_received;
> +}
> +
> +/* Must be called with vq_lock held */
> +static int _viommu_send_reqs_sync(struct viommu_dev *viommu,
> +				  struct viommu_request *req, int nr,
> +				  int *nr_sent)
> +{
> +	int i, ret;
> +	ktime_t timeout;
> +	int nr_received = 0;
> +	struct scatterlist *sg[2];
> +	/*
> +	 * FIXME: as it stands, 1s timeout per request. This is a voluntary
> +	 * exaggeration because I have no idea how real our ktime is. Are we
> +	 * using a RTC? Are we aware of steal time? I don't know much about
> +	 * this, need to do some digging.
> +	 */
> +	unsigned long timeout_ms = 1000;
> +
> +	*nr_sent = 0;
> +
> +	for (i = 0; i < nr; i++, req++) {
> +		/*
> +		 * The backend will allocate one indirect descriptor for each
> +		 * request, which allows to double the ring consumption, but
> +		 * might be slower.
> +		 */
> +		req->written = 0;
> +
> +		sg[0] = &req->head;
> +		sg[1] = &req->tail;
> +
> +		ret = virtqueue_add_sgs(viommu->vq, sg, 1, 1, req,
> +					GFP_ATOMIC);
> +		if (ret)
> +			break;
> +
> +		list_add_tail(&req->list, &viommu->pending_requests);
> +	}
> +
> +	if (i && !virtqueue_kick(viommu->vq))
> +		return -EPIPE;
> +
> +	/*
> +	 * Absolutely no wiggle room here. We're not allowed to sleep as
> callers
> +	 * might be holding spinlocks, so we have to poll like savages until
> +	 * something appears. Hopefully the host already handled the
> request
> +	 * during the above kick and returned it to us.
> +	 *
> +	 * A nice improvement would be for the caller to tell us if we can
> sleep
> +	 * whilst mapping, but this has to go through the IOMMU/DMA API.
> +	 */
> +	timeout = ktime_add_ms(ktime_get(), timeout_ms * i);
> +	while (nr_received < i && ktime_before(ktime_get(), timeout))
{
> +		nr_received += viommu_receive_resp(viommu, i -
> nr_received);
> +		if (nr_received < i) {
> +			/*
> +			 * FIXME: what's a good way to yield to host? A
> second
> +			 * virtqueue_kick won't have any effect since we
> haven't
> +			 * added any descriptor.
> +			 */
> +			udelay(10);
> +		}
> +	}
> +	dev_dbg(viommu->dev, "request took %lld us\n",
> +		ktime_us_delta(ktime_get(), ktime_sub_ms(timeout,
> timeout_ms * i)));
> +
> +	if (nr_received != i)
> +		ret = -ETIMEDOUT;
> +
> +	if (ret == -ENOSPC && nr_received)
> +		/*
> +		 * We've freed some space since virtio told us that the ring is
> +		 * full, tell the caller to come back later (after releasing the
> +		 * lock first, to be fair to other threads)
> +		 */
> +		ret = -EAGAIN;
> +
> +	*nr_sent = nr_received;
> +
> +	return ret;
> +}
> +
> +/**
> + * viommu_send_reqs_sync - add a batch of requests, kick the host and
> wait for
> + *                         them to return
> + *
> + * @req: array of requests
> + * @nr: size of the array
> + * @nr_sent: contains the number of requests actually sent after this
> function
> + *           returns
> + *
> + * Return 0 on success, or an error if we failed to send some of the
> requests.
> + */
> +static int viommu_send_reqs_sync(struct viommu_dev *viommu,
> +				 struct viommu_request *req, int nr,
> +				 int *nr_sent)
> +{
> +	int ret;
> +	int sent = 0;
> +	unsigned long flags;
> +
> +	*nr_sent = 0;
> +	do {
> +		spin_lock_irqsave(&viommu->vq_lock, flags);
> +		ret = _viommu_send_reqs_sync(viommu, req, nr, &sent);
> +		spin_unlock_irqrestore(&viommu->vq_lock, flags);
> +
> +		*nr_sent += sent;
> +		req += sent;
> +		nr -= sent;
> +	} while (ret == -EAGAIN);
> +
> +	return ret;
> +}
> +
> +/**
> + * viommu_send_req_sync - send one request and wait for reply
> + *
> + * @head_ptr: pointer to a virtio_iommu_req_* structure
> + *
> + * Returns 0 if the request was successful, or an error number
> +otherwise. No
> + * distinction is done between transport and request errors.
> + */
> +static int viommu_send_req_sync(struct viommu_dev *viommu, void
> +*head_ptr) {
> +	int ret;
> +	int nr_sent;
> +	struct viommu_request req;
> +	size_t head_size, tail_size;
> +	struct virtio_iommu_req_tail *tail;
> +	struct virtio_iommu_req_head *head = head_ptr;
> +
> +	ret = viommu_get_req_size(head, &head_size, &tail_size);
> +	if (ret)
> +		return ret;
> +
> +	dev_dbg(viommu->dev, "Sending request 0x%x, %zu bytes\n",
> head->type,
> +		head_size + tail_size);
> +
> +	tail = head_ptr + head_size;
> +
> +	sg_init_one(&req.head, head, head_size);
> +	sg_init_one(&req.tail, tail, tail_size);
> +
> +	ret = viommu_send_reqs_sync(viommu, &req, 1, &nr_sent);
> +	if (ret || !req.written || nr_sent != 1) {
> +		dev_err(viommu->dev, "failed to send command\n");
> +		return -EIO;
> +	}
> +
> +	ret = -viommu_status_to_errno(tail->status);
> +
> +	if (ret)
> +		dev_dbg(viommu->dev, " completed with %d\n", ret);
> +
> +	return ret;
> +}
> +
> +static int viommu_tlb_map(struct viommu_domain *vdomain, unsigned
> long iova,
> +			  phys_addr_t paddr, size_t size)
> +{
> +	unsigned long flags;
> +	struct viommu_mapping *mapping;
> +
> +	mapping = kzalloc(sizeof(*mapping), GFP_ATOMIC);
> +	if (!mapping)
> +		return -ENOMEM;
> +
> +	mapping->paddr = paddr;
> +	mapping->iova.start = iova;
> +	mapping->iova.last = iova + size - 1;
> +
> +	spin_lock_irqsave(&vdomain->mappings_lock, flags);
> +	interval_tree_insert(&mapping->iova, &vdomain->mappings);
> +	spin_unlock_irqrestore(&vdomain->mappings_lock, flags);
> +
> +	return 0;
> +}
> +
> +static size_t viommu_tlb_unmap(struct viommu_domain *vdomain,
> +			       unsigned long iova, size_t size) {
> +	size_t unmapped = 0;
> +	unsigned long flags;
> +	unsigned long last = iova + size - 1;
> +	struct viommu_mapping *mapping = NULL;
> +	struct interval_tree_node *node, *next;
> +
> +	spin_lock_irqsave(&vdomain->mappings_lock, flags);
> +	next = interval_tree_iter_first(&vdomain->mappings, iova, last);
> +	while (next) {
> +		node = next;
> +		mapping = container_of(node, struct viommu_mapping,
> iova);
> +
> +		next = interval_tree_iter_next(node, iova, last);
> +
> +		/*
> +		 * Note that for a partial range, this will return the full
> +		 * mapping so we avoid sending split requests to the device.
> +		 */
> +		unmapped += mapping->iova.last - mapping->iova.start + 1;
> +
> +		interval_tree_remove(node, &vdomain->mappings);
> +		kfree(mapping);
> +	}
> +	spin_unlock_irqrestore(&vdomain->mappings_lock, flags);
> +
> +	return unmapped;
> +}
> +
> +/* IOMMU API */
> +
> +static bool viommu_capable(enum iommu_cap cap) {
> +	return false; /* :( */
> +}
> +
> +static struct iommu_domain *viommu_domain_alloc(unsigned type) {
> +	struct viommu_domain *vdomain;
> +
> +	if (type != IOMMU_DOMAIN_UNMANAGED && type !>
IOMMU_DOMAIN_DMA)
> +		return NULL;
> +
> +	vdomain = kzalloc(sizeof(struct viommu_domain), GFP_KERNEL);
> +	if (!vdomain)
> +		return NULL;
> +
> +	vdomain->id >
atomic64_inc_return_relaxed(&viommu_domain_ids_gen);
> +
> +	mutex_init(&vdomain->mutex);
> +	spin_lock_init(&vdomain->mappings_lock);
> +	vdomain->mappings = RB_ROOT;
> +
> +	pr_debug("alloc domain of type %d -> %llu\n", type,
vdomain->id);
> +
> +	if (type == IOMMU_DOMAIN_DMA &&
> +	    iommu_get_dma_cookie(&vdomain->domain)) {
> +		kfree(vdomain);
> +		return NULL;
> +	}
> +
> +	return &vdomain->domain;
> +}
> +
> +static void viommu_domain_free(struct iommu_domain *domain) {
> +	struct viommu_domain *vdomain = to_viommu_domain(domain);
> +
> +	pr_debug("free domain %llu\n", vdomain->id);
> +
> +	iommu_put_dma_cookie(domain);
> +
> +	/* Free all remaining mappings (size 2^64) */
> +	viommu_tlb_unmap(vdomain, 0, 0);
> +
> +	kfree(vdomain);
> +}
> +
> +static int viommu_attach_dev(struct iommu_domain *domain, struct
> device
> +*dev) {
> +	int i;
> +	int ret = 0;
> +	struct iommu_fwspec *fwspec = dev->iommu_fwspec;
> +	struct viommu_endpoint *vdev = fwspec->iommu_priv;
> +	struct viommu_domain *vdomain = to_viommu_domain(domain);
> +	struct virtio_iommu_req_attach req = {
> +		.head.type	= VIRTIO_IOMMU_T_ATTACH,
> +		.address_space	= cpu_to_le32(vdomain->id),
> +	};
> +
> +	mutex_lock(&vdomain->mutex);
> +	if (!vdomain->viommu) {
> +		struct viommu_dev *viommu = vdev->viommu;
> +
> +		vdomain->viommu = viommu;
> +
> +		domain->pgsize_bitmap		= viommu-
> >pgsize_bitmap;
> +		domain->geometry.aperture_start	= viommu-
> >aperture_start;
> +		domain->geometry.aperture_end	= viommu-
> >aperture_end;
> +		domain->geometry.force_aperture	= true;
> +
> +	} else if (vdomain->viommu != vdev->viommu) {
> +		dev_err(dev, "cannot attach to foreign VIOMMU\n");
> +		ret = -EXDEV;
> +	}
> +	mutex_unlock(&vdomain->mutex);
> +
> +	if (ret)
> +		return ret;
> +
> +	/*
> +	 * When attaching the device to a new domain, it will be detached
> from
> +	 * the old one and, if as as a result the old domain isn't attached
to
> +	 * any device, all mappings are removed from the old domain and it is
> +	 * freed. (Note that we can't use get_domain_for_dev here, it
> returns
> +	 * the default domain during initial attach.)
> +	 *
> +	 * Take note of the device disappearing, so we can ignore unmap
> request
> +	 * on stale domains (that is, between this detach and the upcoming
> +	 * free.)
> +	 *
> +	 * vdev->vdomain is protected by group->mutex
> +	 */
> +	if (vdev->vdomain) {
> +		dev_dbg(dev, "detach from domain %llu\n", vdev-
> >vdomain->id);
> +		vdev->vdomain->attached--;
> +	}
> +
> +	dev_dbg(dev, "attach to domain %llu\n", vdomain->id);
> +
> +	for (i = 0; i < fwspec->num_ids; i++) {
> +		req.device = cpu_to_le32(fwspec->ids[i]);
> +
> +		ret = viommu_send_req_sync(vdomain->viommu, &req);
> +		if (ret)
> +			break;
> +	}
> +
> +	vdomain->attached++;
> +	vdev->vdomain = vdomain;
> +
> +	return ret;
> +}
> +
> +static int viommu_map(struct iommu_domain *domain, unsigned long iova,
> +		      phys_addr_t paddr, size_t size, int prot) {
> +	int ret;
> +	struct viommu_domain *vdomain = to_viommu_domain(domain);
> +	struct virtio_iommu_req_map req = {
> +		.head.type	= VIRTIO_IOMMU_T_MAP,
> +		.address_space	= cpu_to_le32(vdomain->id),
> +		.virt_addr	= cpu_to_le64(iova),
> +		.phys_addr	= cpu_to_le64(paddr),
> +		.size		= cpu_to_le64(size),
> +	};
> +
> +	pr_debug("map %llu 0x%lx -> 0x%llx (%zu)\n", vdomain->id,
iova,
> +		 paddr, size);

A query, when I am tracing above prints I see same physical address is mapped
with two different virtual address, do you know why kernel does this?

Thanks
-Bharat

> +
> +	if (!vdomain->attached)
> +		return -ENODEV;
> +
> +	if (prot & IOMMU_READ)
> +		req.flags |= cpu_to_le32(VIRTIO_IOMMU_MAP_F_READ);
> +
> +	if (prot & IOMMU_WRITE)
> +		req.flags |= cpu_to_le32(VIRTIO_IOMMU_MAP_F_WRITE);
> +
> +	ret = viommu_tlb_map(vdomain, iova, paddr, size);
> +	if (ret)
> +		return ret;
> +
> +	ret = viommu_send_req_sync(vdomain->viommu, &req);
> +	if (ret)
> +		viommu_tlb_unmap(vdomain, iova, size);
> +
> +	return ret;
> +}
> +
> +static size_t viommu_unmap(struct iommu_domain *domain, unsigned
> long iova,
> +			   size_t size)
> +{
> +	int ret;
> +	size_t unmapped;
> +	struct viommu_domain *vdomain = to_viommu_domain(domain);
> +	struct virtio_iommu_req_unmap req = {
> +		.head.type	= VIRTIO_IOMMU_T_UNMAP,
> +		.address_space	= cpu_to_le32(vdomain->id),
> +		.virt_addr	= cpu_to_le64(iova),
> +	};
> +
> +	pr_debug("unmap %llu 0x%lx (%zu)\n", vdomain->id, iova,
size);
> +
> +	/* Callers may unmap after detach, but device already took care of it.
> */
> +	if (!vdomain->attached)
> +		return size;
> +
> +	unmapped = viommu_tlb_unmap(vdomain, iova, size);
> +	if (unmapped < size)
> +		return 0;
> +
> +	req.size = cpu_to_le64(unmapped);
> +
> +	ret = viommu_send_req_sync(vdomain->viommu, &req);
> +	if (ret)
> +		return 0;
> +
> +	return unmapped;
> +}
> +
> +static size_t viommu_map_sg(struct iommu_domain *domain, unsigned
> long iova,
> +			    struct scatterlist *sg, unsigned int nents, int prot) {
> +	int i, ret;
> +	int nr_sent;
> +	size_t mapped;
> +	size_t min_pagesz;
> +	size_t total_size;
> +	struct scatterlist *s;
> +	unsigned int flags = 0;
> +	unsigned long cur_iova;
> +	unsigned long mapped_iova;
> +	size_t head_size, tail_size;
> +	struct viommu_request reqs[nents];
> +	struct virtio_iommu_req_map map_reqs[nents];
> +	struct viommu_domain *vdomain = to_viommu_domain(domain);
> +
> +	if (!vdomain->attached)
> +		return 0;
> +
> +	pr_debug("map_sg %llu %u 0x%lx\n", vdomain->id, nents,
iova);
> +
> +	if (prot & IOMMU_READ)
> +		flags |= VIRTIO_IOMMU_MAP_F_READ;
> +
> +	if (prot & IOMMU_WRITE)
> +		flags |= VIRTIO_IOMMU_MAP_F_WRITE;
> +
> +	min_pagesz = 1 << __ffs(domain->pgsize_bitmap);
> +	tail_size = sizeof(struct virtio_iommu_req_tail);
> +	head_size = sizeof(*map_reqs) - tail_size;
> +
> +	cur_iova = iova;
> +
> +	for_each_sg(sg, s, nents, i) {
> +		size_t size = s->length;
> +		phys_addr_t paddr = sg_phys(s);
> +		void *tail = (void *)&map_reqs[i] + head_size;
> +
> +		if (!IS_ALIGNED(paddr | size, min_pagesz)) {
> +			ret = -EFAULT;
> +			break;
> +		}
> +
> +		/* TODO: merge physically-contiguous mappings if any */
> +		map_reqs[i] = (struct virtio_iommu_req_map) {
> +			.head.type	= VIRTIO_IOMMU_T_MAP,
> +			.address_space	= cpu_to_le32(vdomain->id),
> +			.flags		= cpu_to_le32(flags),
> +			.virt_addr	= cpu_to_le64(cur_iova),
> +			.phys_addr	= cpu_to_le64(paddr),
> +			.size		= cpu_to_le64(size),
> +		};
> +
> +		ret = viommu_tlb_map(vdomain, cur_iova, paddr, size);
> +		if (ret)
> +			break;
> +
> +		sg_init_one(&reqs[i].head, &map_reqs[i], head_size);
> +		sg_init_one(&reqs[i].tail, tail, tail_size);
> +
> +		cur_iova += size;
> +	}
> +
> +	total_size = cur_iova - iova;
> +
> +	if (ret) {
> +		viommu_tlb_unmap(vdomain, iova, total_size);
> +		return 0;
> +	}
> +
> +	ret = viommu_send_reqs_sync(vdomain->viommu, reqs, i,
> &nr_sent);
> +
> +	if (nr_sent != nents)
> +		goto err_rollback;
> +
> +	for (i = 0; i < nents; i++) {
> +		if (!reqs[i].written || map_reqs[i].tail.status)
> +			goto err_rollback;
> +	}
> +
> +	return total_size;
> +
> +err_rollback:
> +	/*
> +	 * Any request in the range might have failed. Unmap what was
> +	 * successful.
> +	 */
> +	cur_iova = iova;
> +	mapped_iova = iova;
> +	mapped = 0;
> +	for_each_sg(sg, s, nents, i) {
> +		size_t size = s->length;
> +
> +		cur_iova += size;
> +
> +		if (!reqs[i].written || map_reqs[i].tail.status) {
> +			if (mapped)
> +				viommu_unmap(domain, mapped_iova,
> mapped);
> +
> +			mapped_iova = cur_iova;
> +			mapped = 0;
> +		} else {
> +			mapped += size;
> +		}
> +	}
> +
> +	viommu_tlb_unmap(vdomain, iova, total_size);
> +
> +	return 0;
> +}
> +
> +static phys_addr_t viommu_iova_to_phys(struct iommu_domain *domain,
> +				       dma_addr_t iova)
> +{
> +	u64 paddr = 0;
> +	unsigned long flags;
> +	struct viommu_mapping *mapping;
> +	struct interval_tree_node *node;
> +	struct viommu_domain *vdomain = to_viommu_domain(domain);
> +
> +	spin_lock_irqsave(&vdomain->mappings_lock, flags);
> +	node = interval_tree_iter_first(&vdomain->mappings, iova, iova);
> +	if (node) {
> +		mapping = container_of(node, struct viommu_mapping,
> iova);
> +		paddr = mapping->paddr + (iova - mapping->iova.start);
> +	}
> +	spin_unlock_irqrestore(&vdomain->mappings_lock, flags);
> +
> +	pr_debug("iova_to_phys %llu 0x%llx->0x%llx\n",
vdomain->id, iova,
> +		 paddr);
> +
> +	return paddr;
> +}
> +
> +static struct iommu_ops viommu_ops;
> +static struct virtio_driver virtio_iommu_drv;
> +
> +static int viommu_match_node(struct device *dev, void *data) {
> +	return dev->parent->fwnode == data;
> +}
> +
> +static struct viommu_dev *viommu_get_by_fwnode(struct
> fwnode_handle
> +*fwnode) {
> +	struct device *dev = driver_find_device(&virtio_iommu_drv.driver,
> NULL,
> +						fwnode,
> viommu_match_node);
> +	put_device(dev);
> +
> +	return dev ? dev_to_virtio(dev)->priv : NULL; }
> +
> +static int viommu_add_device(struct device *dev) {
> +	struct iommu_group *group;
> +	struct viommu_endpoint *vdev;
> +	struct viommu_dev *viommu = NULL;
> +	struct iommu_fwspec *fwspec = dev->iommu_fwspec;
> +
> +	if (!fwspec || fwspec->ops != &viommu_ops)
> +		return -ENODEV;
> +
> +	viommu = viommu_get_by_fwnode(fwspec->iommu_fwnode);
> +	if (!viommu)
> +		return -ENODEV;
> +
> +	vdev = kzalloc(sizeof(*vdev), GFP_KERNEL);
> +	if (!vdev)
> +		return -ENOMEM;
> +
> +	vdev->viommu = viommu;
> +	fwspec->iommu_priv = vdev;
> +
> +	/*
> +	 * Last step creates a default domain and attaches to it. Everything
> +	 * must be ready.
> +	 */
> +	group = iommu_group_get_for_dev(dev);
> +
> +	return PTR_ERR_OR_ZERO(group);
> +}
> +
> +static void viommu_remove_device(struct device *dev) {
> +	kfree(dev->iommu_fwspec->iommu_priv);
> +}
> +
> +static struct iommu_group *
> +viommu_device_group(struct device *dev) {
> +	if (dev_is_pci(dev))
> +		return pci_device_group(dev);
> +	else
> +		return generic_device_group(dev);
> +}
> +
> +static int viommu_of_xlate(struct device *dev, struct of_phandle_args
> +*args) {
> +	u32 *id = args->args;
> +
> +	dev_dbg(dev, "of_xlate 0x%x\n", *id);
> +	return iommu_fwspec_add_ids(dev, args->args, 1); }
> +
> +/*
> + * (Maybe) temporary hack for device pass-through into guest userspace.
> +On ARM
> + * with an ITS, VFIO will look for a region where to map the doorbell,
> +even
> + * though the virtual doorbell is never written to by the device, and
> +instead
> + * the host injects interrupts directly. TODO: sort this out in VFIO.
> + */
> +#define MSI_IOVA_BASE			0x8000000
> +#define MSI_IOVA_LENGTH			0x100000
> +
> +static void viommu_get_resv_regions(struct device *dev, struct
> +list_head *head) {
> +	struct iommu_resv_region *region;
> +	int prot = IOMMU_WRITE | IOMMU_NOEXEC | IOMMU_MMIO;
> +
> +	region = iommu_alloc_resv_region(MSI_IOVA_BASE,
> MSI_IOVA_LENGTH, prot,
> +					 IOMMU_RESV_MSI);
> +	if (!region)
> +		return;
> +
> +	list_add_tail(&region->list, head);
> +}
> +
> +static void viommu_put_resv_regions(struct device *dev, struct
> +list_head *head) {
> +	struct iommu_resv_region *entry, *next;
> +
> +	list_for_each_entry_safe(entry, next, head, list)
> +		kfree(entry);
> +}
> +
> +static struct iommu_ops viommu_ops = {
> +	.capable		= viommu_capable,
> +	.domain_alloc		= viommu_domain_alloc,
> +	.domain_free		= viommu_domain_free,
> +	.attach_dev		= viommu_attach_dev,
> +	.map			= viommu_map,
> +	.unmap			= viommu_unmap,
> +	.map_sg			= viommu_map_sg,
> +	.iova_to_phys		= viommu_iova_to_phys,
> +	.add_device		= viommu_add_device,
> +	.remove_device		= viommu_remove_device,
> +	.device_group		= viommu_device_group,
> +	.of_xlate		= viommu_of_xlate,
> +	.get_resv_regions	= viommu_get_resv_regions,
> +	.put_resv_regions	= viommu_put_resv_regions,
> +};
> +
> +static int viommu_init_vq(struct viommu_dev *viommu) {
> +	struct virtio_device *vdev = dev_to_virtio(viommu->dev);
> +	vq_callback_t *callback = NULL;
> +	const char *name = "request";
> +	int ret;
> +
> +	ret = vdev->config->find_vqs(vdev, 1, &viommu->vq,
&callback,
> +				     &name, NULL);
> +	if (ret)
> +		dev_err(viommu->dev, "cannot find VQ\n");
> +
> +	return ret;
> +}
> +
> +static int viommu_probe(struct virtio_device *vdev) {
> +	struct device *parent_dev = vdev->dev.parent;
> +	struct viommu_dev *viommu = NULL;
> +	struct device *dev = &vdev->dev;
> +	int ret;
> +
> +	viommu = kzalloc(sizeof(*viommu), GFP_KERNEL);
> +	if (!viommu)
> +		return -ENOMEM;
> +
> +	spin_lock_init(&viommu->vq_lock);
> +	INIT_LIST_HEAD(&viommu->pending_requests);
> +	viommu->dev = dev;
> +	viommu->vdev = vdev;
> +
> +	ret = viommu_init_vq(viommu);
> +	if (ret)
> +		goto err_free_viommu;
> +
> +	virtio_cread(vdev, struct virtio_iommu_config, page_sizes,
> +		     &viommu->pgsize_bitmap);
> +
> +	viommu->aperture_end = -1UL;
> +
> +	virtio_cread_feature(vdev, VIRTIO_IOMMU_F_INPUT_RANGE,
> +			     struct virtio_iommu_config, input_range.start,
> +			     &viommu->aperture_start);
> +
> +	virtio_cread_feature(vdev, VIRTIO_IOMMU_F_INPUT_RANGE,
> +			     struct virtio_iommu_config, input_range.end,
> +			     &viommu->aperture_end);
> +
> +	if (!viommu->pgsize_bitmap) {
> +		ret = -EINVAL;
> +		goto err_free_viommu;
> +	}
> +
> +	viommu_ops.pgsize_bitmap = viommu->pgsize_bitmap;
> +
> +	/*
> +	 * Not strictly necessary, virtio would enable it later. This allows to
> +	 * start using the request queue early.
> +	 */
> +	virtio_device_ready(vdev);
> +
> +	ret = iommu_device_sysfs_add(&viommu->iommu, dev, NULL,
"%s",
> +				     virtio_bus_name(vdev));
> +	if (ret)
> +		goto err_free_viommu;
> +
> +	iommu_device_set_ops(&viommu->iommu, &viommu_ops);
> +	iommu_device_set_fwnode(&viommu->iommu, parent_dev-
> >fwnode);
> +
> +	iommu_device_register(&viommu->iommu);
> +
> +#ifdef CONFIG_PCI
> +	if (pci_bus_type.iommu_ops != &viommu_ops) {
> +		pci_request_acs();
> +		ret = bus_set_iommu(&pci_bus_type, &viommu_ops);
> +		if (ret)
> +			goto err_unregister;
> +	}
> +#endif
> +#ifdef CONFIG_ARM_AMBA
> +	if (amba_bustype.iommu_ops != &viommu_ops) {
> +		ret = bus_set_iommu(&amba_bustype, &viommu_ops);
> +		if (ret)
> +			goto err_unregister;
> +	}
> +#endif
> +	if (platform_bus_type.iommu_ops != &viommu_ops) {
> +		ret = bus_set_iommu(&platform_bus_type, &viommu_ops);
> +		if (ret)
> +			goto err_unregister;
> +	}
> +
> +	vdev->priv = viommu;
> +
> +	dev_info(viommu->dev, "probe successful\n");
> +
> +	return 0;
> +
> +err_unregister:
> +	iommu_device_unregister(&viommu->iommu);
> +
> +err_free_viommu:
> +	kfree(viommu);
> +
> +	return ret;
> +}
> +
> +static void viommu_remove(struct virtio_device *vdev) {
> +	struct viommu_dev *viommu = vdev->priv;
> +
> +	iommu_device_unregister(&viommu->iommu);
> +	kfree(viommu);
> +
> +	dev_info(&vdev->dev, "device removed\n"); }
> +
> +static void viommu_config_changed(struct virtio_device *vdev) {
> +	dev_warn(&vdev->dev, "config changed\n"); }
> +
> +static unsigned int features[] = {
> +	VIRTIO_IOMMU_F_INPUT_RANGE,
> +};
> +
> +static struct virtio_device_id id_table[] = {
> +	{ VIRTIO_ID_IOMMU, VIRTIO_DEV_ANY_ID },
> +	{ 0 },
> +};
> +
> +static struct virtio_driver virtio_iommu_drv = {
> +	.driver.name		= KBUILD_MODNAME,
> +	.driver.owner		= THIS_MODULE,
> +	.id_table		= id_table,
> +	.feature_table		= features,
> +	.feature_table_size	= ARRAY_SIZE(features),
> +	.probe			= viommu_probe,
> +	.remove			= viommu_remove,
> +	.config_changed		= viommu_config_changed,
> +};
> +
> +module_virtio_driver(virtio_iommu_drv);
> +
> +IOMMU_OF_DECLARE(viommu, "virtio,mmio", NULL);
> +
> +MODULE_DESCRIPTION("virtio-iommu driver");
MODULE_AUTHOR("Jean-
> Philippe
> +Brucker <jean-philippe.brucker at arm.com>");
> +MODULE_LICENSE("GPL v2");
> diff --git a/include/uapi/linux/Kbuild b/include/uapi/linux/Kbuild index
> 1f25c86374ad..c0cb0f173258 100644
> --- a/include/uapi/linux/Kbuild
> +++ b/include/uapi/linux/Kbuild
> @@ -467,6 +467,7 @@ header-y += virtio_console.h  header-y +>
virtio_gpu.h  header-y += virtio_ids.h  header-y += virtio_input.h
> +header-y += virtio_iommu.h
>  header-y += virtio_mmio.h
>  header-y += virtio_net.h
>  header-y += virtio_pci.h
> diff --git a/include/uapi/linux/virtio_ids.h
b/include/uapi/linux/virtio_ids.h
> index 6d5c3b2d4f4d..934ed3d3cd3f 100644
> --- a/include/uapi/linux/virtio_ids.h
> +++ b/include/uapi/linux/virtio_ids.h
> @@ -43,5 +43,6 @@
>  #define VIRTIO_ID_INPUT        18 /* virtio input */
>  #define VIRTIO_ID_VSOCK        19 /* virtio vsock transport */
>  #define VIRTIO_ID_CRYPTO       20 /* virtio crypto */
> +#define VIRTIO_ID_IOMMU	    61216 /* virtio IOMMU (temporary) */
>
>  #endif /* _LINUX_VIRTIO_IDS_H */
> diff --git a/include/uapi/linux/virtio_iommu.h
> b/include/uapi/linux/virtio_iommu.h
> new file mode 100644
> index 000000000000..ec74c9a727d4
> --- /dev/null
> +++ b/include/uapi/linux/virtio_iommu.h
> @@ -0,0 +1,142 @@
> +/*
> + * Copyright (C) 2017 ARM Ltd.
> + *
> + * This header is BSD licensed so anyone can use the definitions
> + * to implement compatible drivers/servers:
> + *
> + * Redistribution and use in source and binary forms, with or without
> + * modification, are permitted provided that the following conditions
> + * are met:
> + * 1. Redistributions of source code must retain the above copyright
> + *    notice, this list of conditions and the following disclaimer.
> + * 2. Redistributions in binary form must reproduce the above copyright
> + *    notice, this list of conditions and the following disclaimer in the
> + *    documentation and/or other materials provided with the distribution.
> + * 3. Neither the name of ARM Ltd. nor the names of its contributors
> + *    may be used to endorse or promote products derived from this
> software
> + *    without specific prior written permission.
> + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
> CONTRIBUTORS
> + * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT
> NOT
> + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
> FITNESS
> + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL IBM
> OR
> + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
> + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
> NOT
> + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
> OF
> + * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
> CAUSED AND
> + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
> + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
> OUT
> + * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
> + * SUCH DAMAGE.
> + */
> +#ifndef _UAPI_LINUX_VIRTIO_IOMMU_H
> +#define _UAPI_LINUX_VIRTIO_IOMMU_H
> +
> +/* Feature bits */
> +#define VIRTIO_IOMMU_F_INPUT_RANGE		0
> +#define VIRTIO_IOMMU_F_IOASID_BITS		1
> +#define VIRTIO_IOMMU_F_MAP_UNMAP		2
> +#define VIRTIO_IOMMU_F_BYPASS			3
> +
> +__packed
> +struct virtio_iommu_config {
> +	/* Supported page sizes */
> +	__u64					page_sizes;
> +	struct virtio_iommu_range {
> +		__u64				start;
> +		__u64				end;
> +	} input_range;
> +	__u8 					ioasid_bits;
> +};
> +
> +/* Request types */
> +#define VIRTIO_IOMMU_T_ATTACH			0x01
> +#define VIRTIO_IOMMU_T_DETACH			0x02
> +#define VIRTIO_IOMMU_T_MAP			0x03
> +#define VIRTIO_IOMMU_T_UNMAP			0x04
> +
> +/* Status types */
> +#define VIRTIO_IOMMU_S_OK			0x00
> +#define VIRTIO_IOMMU_S_IOERR			0x01
> +#define VIRTIO_IOMMU_S_UNSUPP			0x02
> +#define VIRTIO_IOMMU_S_DEVERR			0x03
> +#define VIRTIO_IOMMU_S_INVAL			0x04
> +#define VIRTIO_IOMMU_S_RANGE			0x05
> +#define VIRTIO_IOMMU_S_NOENT			0x06
> +#define VIRTIO_IOMMU_S_FAULT			0x07
> +
> +__packed
> +struct virtio_iommu_req_head {
> +	__u8					type;
> +	__u8					reserved[3];
> +};
> +
> +__packed
> +struct virtio_iommu_req_tail {
> +	__u8					status;
> +	__u8					reserved[3];
> +};
> +
> +__packed
> +struct virtio_iommu_req_attach {
> +	struct virtio_iommu_req_head		head;
> +
> +	__le32					address_space;
> +	__le32					device;
> +	__le32					reserved;
> +
> +	struct virtio_iommu_req_tail		tail;
> +};
> +
> +__packed
> +struct virtio_iommu_req_detach {
> +	struct virtio_iommu_req_head		head;
> +
> +	__le32					device;
> +	__le32					reserved;
> +
> +	struct virtio_iommu_req_tail		tail;
> +};
> +
> +#define VIRTIO_IOMMU_MAP_F_READ			(1 << 0)
> +#define VIRTIO_IOMMU_MAP_F_WRITE		(1 << 1)
> +#define VIRTIO_IOMMU_MAP_F_EXEC			(1 << 2)
> +
> +#define VIRTIO_IOMMU_MAP_F_MASK
> 	(VIRTIO_IOMMU_MAP_F_READ |	\
> +
> VIRTIO_IOMMU_MAP_F_WRITE |	\
> +
> VIRTIO_IOMMU_MAP_F_EXEC)
> +
> +__packed
> +struct virtio_iommu_req_map {
> +	struct virtio_iommu_req_head		head;
> +
> +	__le32					address_space;
> +	__le32					flags;
> +	__le64					virt_addr;
> +	__le64					phys_addr;
> +	__le64					size;
> +
> +	struct virtio_iommu_req_tail		tail;
> +};
> +
> +__packed
> +struct virtio_iommu_req_unmap {
> +	struct virtio_iommu_req_head		head;
> +
> +	__le32					address_space;
> +	__le32					flags;
> +	__le64					virt_addr;
> +	__le64					size;
> +
> +	struct virtio_iommu_req_tail		tail;
> +};
> +
> +union virtio_iommu_req {
> +	struct virtio_iommu_req_head		head;
> +
> +	struct virtio_iommu_req_attach		attach;
> +	struct virtio_iommu_req_detach		detach;
> +	struct virtio_iommu_req_map		map;
> +	struct virtio_iommu_req_unmap		unmap;
> +};
> +
> +#endif
> --
> 2.12.1
>
>
> ---------------------------------------------------------------------
> To unsubscribe, e-mail: virtio-dev-unsubscribe at lists.oasis-open.org
> For additional commands, e-mail: virtio-dev-help at lists.oasis-open.org