Linux Virtualization - Jan 2013 - [PATCH v2 0/3] tcm_vhost: Optimize gup in vhost_scsi_map_to_sgl

Changes in v2:
- Use GFP_KERNEL instead of GFP_ATOMIC
- Handle pages pinned less than wanted

Asias He (3):
  tcm_vhost: Introduce iov_num_pages
  tcm_vhost: Use iov_num_pages to calculate sgl_count
  tcm_vhost: Optimize gup in vhost_scsi_map_to_sgl

 drivers/vhost/tcm_vhost.c | 68 +++++++++++++++++++++++++++--------------------
 1 file changed, 39 insertions(+), 29 deletions(-)

-- 
1.8.1

Add a helper to calculate the number of pages needed for a iov entry.

Signed-off-by: Asias He <asias at redhat.com>
---
 drivers/vhost/tcm_vhost.c | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/drivers/vhost/tcm_vhost.c b/drivers/vhost/tcm_vhost.c
index 3720604..ca35c16 100644
--- a/drivers/vhost/tcm_vhost.c
+++ b/drivers/vhost/tcm_vhost.c
@@ -77,6 +77,12 @@ static struct workqueue_struct *tcm_vhost_workqueue;
 static DEFINE_MUTEX(tcm_vhost_mutex);
 static LIST_HEAD(tcm_vhost_list);
 
+static inline int iov_num_pages(struct iovec *iov)
+{
+	return (PAGE_ALIGN((unsigned long)iov->iov_base + iov->iov_len) -
+	       ((unsigned long)iov->iov_base & PAGE_MASK)) >> PAGE_SHIFT;
+}
+
 static int tcm_vhost_check_true(struct se_portal_group *se_tpg)
 {
 	return 1;
-- 
1.8.1

Signed-off-by: Asias He <asias at redhat.com>
---
 drivers/vhost/tcm_vhost.c | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/drivers/vhost/tcm_vhost.c b/drivers/vhost/tcm_vhost.c
index ca35c16..796a6b0 100644
--- a/drivers/vhost/tcm_vhost.c
+++ b/drivers/vhost/tcm_vhost.c
@@ -479,11 +479,9 @@ static int vhost_scsi_map_iov_to_sgl(struct tcm_vhost_cmd
*tv_cmd,
 	 * Find out how long sglist needs to be
 	 */
 	sgl_count = 0;
-	for (i = 0; i < niov; i++) {
-		sgl_count += (((uintptr_t)iov[i].iov_base + iov[i].iov_len +
-				PAGE_SIZE - 1) >> PAGE_SHIFT) -
-				((uintptr_t)iov[i].iov_base >> PAGE_SHIFT);
-	}
+	for (i = 0; i < niov; i++)
+		sgl_count += iov_num_pages(&iov[i]);
+
 	/* TODO overflow checking */
 
 	sg = kmalloc(sizeof(tv_cmd->tvc_sgl[0]) * sgl_count, GFP_ATOMIC);
-- 
1.8.1

We can get all the pages in one time instead of calling
gup N times.

Signed-off-by: Asias He <asias at redhat.com>
---
 drivers/vhost/tcm_vhost.c | 54 ++++++++++++++++++++++++++---------------------
 1 file changed, 30 insertions(+), 24 deletions(-)

diff --git a/drivers/vhost/tcm_vhost.c b/drivers/vhost/tcm_vhost.c
index 796a6b0..c994f00 100644
--- a/drivers/vhost/tcm_vhost.c
+++ b/drivers/vhost/tcm_vhost.c
@@ -430,40 +430,47 @@ static struct tcm_vhost_cmd *vhost_scsi_allocate_cmd(
  * Returns the number of scatterlist entries used or -errno on error.
  */
 static int vhost_scsi_map_to_sgl(struct scatterlist *sgl,
-	unsigned int sgl_count, void __user *ptr, size_t len, int write)
+	unsigned int sgl_count, struct iovec *iov, int write)
 {
+	unsigned int npages = 0, pages_nr, offset, nbytes;
 	struct scatterlist *sg = sgl;
-	unsigned int npages = 0;
-	int ret;
+	void __user *ptr = iov->iov_base;
+	size_t len = iov->iov_len;
+	struct page **pages;
+	int ret, i;
 
-	while (len > 0) {
-		struct page *page;
-		unsigned int offset = (uintptr_t)ptr & ~PAGE_MASK;
-		unsigned int nbytes = min_t(unsigned int,
-				PAGE_SIZE - offset, len);
+	pages_nr = iov_num_pages(iov);
+	if (pages_nr > sgl_count)
+		return -ENOBUFS;
 
-		if (npages == sgl_count) {
-			ret = -ENOBUFS;
-			goto err;
-		}
+	pages = kmalloc(pages_nr * sizeof(struct page *), GFP_KERNEL);
+	if (!pages)
+		return -ENOMEM;
 
-		ret = get_user_pages_fast((unsigned long)ptr, 1, write, &page);
-		BUG_ON(ret == 0); /* we should either get our page or fail */
-		if (ret < 0)
-			goto err;
+	ret = get_user_pages_fast((unsigned long)ptr, pages_nr, write, pages);
+	/* No pages were pinned */
+	if (ret < 0)
+		goto out;
+	/* Less pages pinned than wanted */
+	if (ret != pages_nr) {
+		for (i = 0; i < ret; i++)
+			put_page(pages[i]);
+		ret = -EFAULT;
+		goto out;
+	}
 
-		sg_set_page(sg, page, nbytes, offset);
+	while (len > 0) {
+		offset = (uintptr_t)ptr & ~PAGE_MASK;
+		nbytes = min_t(unsigned int, PAGE_SIZE - offset, len);
+		sg_set_page(sg, pages[npages], nbytes, offset);
 		ptr += nbytes;
 		len -= nbytes;
 		sg++;
 		npages++;
 	}
-	return npages;
 
-err:
-	/* Put pages that we hold */
-	for (sg = sgl; sg != &sgl[npages]; sg++)
-		put_page(sg_page(sg));
+out:
+	kfree(pages);
 	return ret;
 }
 
@@ -496,8 +503,7 @@ static int vhost_scsi_map_iov_to_sgl(struct tcm_vhost_cmd
*tv_cmd,
 
 	pr_debug("Mapping %u iovecs for %u pages\n", niov, sgl_count);
 	for (i = 0; i < niov; i++) {
-		ret = vhost_scsi_map_to_sgl(sg, sgl_count, iov[i].iov_base,
-					iov[i].iov_len, write);
+		ret = vhost_scsi_map_to_sgl(sg, sgl_count, &iov[i], write);
 		if (ret < 0) {
 			for (i = 0; i < tv_cmd->tvc_sgl_count; i++)
 				put_page(sg_page(&tv_cmd->tvc_sgl[i]));
-- 
1.8.1

Hi Asias,

On Tue, 2013-01-22 at 11:20 +0800, Asias He wrote:
> Add a helper to calculate the number of pages needed for a iov entry.
> 
> Signed-off-by: Asias He <asias at redhat.com>
> ---
Apologies for the long delay.  Applied to target-pending/for-next with a
minor nit below.

Thank you,

--nab
>  drivers/vhost/tcm_vhost.c | 6 ++++++
>  1 file changed, 6 insertions(+)
> 
> diff --git a/drivers/vhost/tcm_vhost.c b/drivers/vhost/tcm_vhost.c
> index 3720604..ca35c16 100644
> --- a/drivers/vhost/tcm_vhost.c
> +++ b/drivers/vhost/tcm_vhost.c
> @@ -77,6 +77,12 @@ static struct workqueue_struct *tcm_vhost_workqueue;
>  static DEFINE_MUTEX(tcm_vhost_mutex);
>  static LIST_HEAD(tcm_vhost_list);
>  
> +static inline int iov_num_pages(struct iovec *iov)
Dropping the unnecessary inline here.
> +{
> +	return (PAGE_ALIGN((unsigned long)iov->iov_base + iov->iov_len) -
> +	       ((unsigned long)iov->iov_base & PAGE_MASK)) >>
PAGE_SHIFT;
> +}
> +
>  static int tcm_vhost_check_true(struct se_portal_group *se_tpg)
>  {
>  	return 1;

On Tue, 2013-01-22 at 11:20 +0800, Asias He wrote:
> Signed-off-by: Asias He <asias at redhat.com>
> ---
Applied to target-pending/for-next.

Thanks!

--nab
>  drivers/vhost/tcm_vhost.c | 8 +++-----
>  1 file changed, 3 insertions(+), 5 deletions(-)
> 
> diff --git a/drivers/vhost/tcm_vhost.c b/drivers/vhost/tcm_vhost.c
> index ca35c16..796a6b0 100644
> --- a/drivers/vhost/tcm_vhost.c
> +++ b/drivers/vhost/tcm_vhost.c
> @@ -479,11 +479,9 @@ static int vhost_scsi_map_iov_to_sgl(struct
tcm_vhost_cmd *tv_cmd,
>  	 * Find out how long sglist needs to be
>  	 */
>  	sgl_count = 0;
> -	for (i = 0; i < niov; i++) {
> -		sgl_count += (((uintptr_t)iov[i].iov_base + iov[i].iov_len +
> -				PAGE_SIZE - 1) >> PAGE_SHIFT) -
> -				((uintptr_t)iov[i].iov_base >> PAGE_SHIFT);
> -	}
> +	for (i = 0; i < niov; i++)
> +		sgl_count += iov_num_pages(&iov[i]);
> +
>  	/* TODO overflow checking */
>  
>  	sg = kmalloc(sizeof(tv_cmd->tvc_sgl[0]) * sgl_count, GFP_ATOMIC);

On Tue, 2013-01-22 at 11:20 +0800, Asias He wrote:
> We can get all the pages in one time instead of calling
> gup N times.
> 
> Signed-off-by: Asias He <asias at redhat.com>
> ---
Everything looks fine after initial testing, so also applied to
target-pending/for-next.

Nice optimization.  :)

--nab
>  drivers/vhost/tcm_vhost.c | 54
++++++++++++++++++++++++++---------------------
>  1 file changed, 30 insertions(+), 24 deletions(-)
> 
> diff --git a/drivers/vhost/tcm_vhost.c b/drivers/vhost/tcm_vhost.c
> index 796a6b0..c994f00 100644
> --- a/drivers/vhost/tcm_vhost.c
> +++ b/drivers/vhost/tcm_vhost.c
> @@ -430,40 +430,47 @@ static struct tcm_vhost_cmd *vhost_scsi_allocate_cmd(
>   * Returns the number of scatterlist entries used or -errno on error.
>   */
>  static int vhost_scsi_map_to_sgl(struct scatterlist *sgl,
> -	unsigned int sgl_count, void __user *ptr, size_t len, int write)
> +	unsigned int sgl_count, struct iovec *iov, int write)
>  {
> +	unsigned int npages = 0, pages_nr, offset, nbytes;
>  	struct scatterlist *sg = sgl;
> -	unsigned int npages = 0;
> -	int ret;
> +	void __user *ptr = iov->iov_base;
> +	size_t len = iov->iov_len;
> +	struct page **pages;
> +	int ret, i;
>  
> -	while (len > 0) {
> -		struct page *page;
> -		unsigned int offset = (uintptr_t)ptr & ~PAGE_MASK;
> -		unsigned int nbytes = min_t(unsigned int,
> -				PAGE_SIZE - offset, len);
> +	pages_nr = iov_num_pages(iov);
> +	if (pages_nr > sgl_count)
> +		return -ENOBUFS;
>  
> -		if (npages == sgl_count) {
> -			ret = -ENOBUFS;
> -			goto err;
> -		}
> +	pages = kmalloc(pages_nr * sizeof(struct page *), GFP_KERNEL);
> +	if (!pages)
> +		return -ENOMEM;
>  
> -		ret = get_user_pages_fast((unsigned long)ptr, 1, write, &page);
> -		BUG_ON(ret == 0); /* we should either get our page or fail */
> -		if (ret < 0)
> -			goto err;
> +	ret = get_user_pages_fast((unsigned long)ptr, pages_nr, write, pages);
> +	/* No pages were pinned */
> +	if (ret < 0)
> +		goto out;
> +	/* Less pages pinned than wanted */
> +	if (ret != pages_nr) {
> +		for (i = 0; i < ret; i++)
> +			put_page(pages[i]);
> +		ret = -EFAULT;
> +		goto out;
> +	}
>  
> -		sg_set_page(sg, page, nbytes, offset);
> +	while (len > 0) {
> +		offset = (uintptr_t)ptr & ~PAGE_MASK;
> +		nbytes = min_t(unsigned int, PAGE_SIZE - offset, len);
> +		sg_set_page(sg, pages[npages], nbytes, offset);
>  		ptr += nbytes;
>  		len -= nbytes;
>  		sg++;
>  		npages++;
>  	}
> -	return npages;
>  
> -err:
> -	/* Put pages that we hold */
> -	for (sg = sgl; sg != &sgl[npages]; sg++)
> -		put_page(sg_page(sg));
> +out:
> +	kfree(pages);
>  	return ret;
>  }
>  
> @@ -496,8 +503,7 @@ static int vhost_scsi_map_iov_to_sgl(struct
tcm_vhost_cmd *tv_cmd,
>  
>  	pr_debug("Mapping %u iovecs for %u pages\n", niov, sgl_count);
>  	for (i = 0; i < niov; i++) {
> -		ret = vhost_scsi_map_to_sgl(sg, sgl_count, iov[i].iov_base,
> -					iov[i].iov_len, write);
> +		ret = vhost_scsi_map_to_sgl(sg, sgl_count, &iov[i], write);
>  		if (ret < 0) {
>  			for (i = 0; i < tv_cmd->tvc_sgl_count; i++)
>  				put_page(sg_page(&tv_cmd->tvc_sgl[i]));

On Tue, Jan 29, 2013 at 10:53:33AM -0800, Nicholas A. Bellinger
wrote:
> On Tue, 2013-01-22 at 11:20 +0800, Asias He wrote:
> > We can get all the pages in one time instead of calling
> > gup N times.
> > 
> > Signed-off-by: Asias He <asias at redhat.com>
> > ---
> 
> Everything looks fine after initial testing, so also applied to
> target-pending/for-next.
> 
> Nice optimization.  :)
> 
> --nab
Maybe best if you put the llist optimization on that branch too then?

On Tue, 2013-01-29 at 21:16 +0200, Michael S. Tsirkin
wrote:
> On Tue, Jan 29, 2013 at 10:53:33AM -0800, Nicholas A. Bellinger wrote:
> > On Tue, 2013-01-22 at 11:20 +0800, Asias He wrote:
> > > We can get all the pages in one time instead of calling
> > > gup N times.
> > > 
> > > Signed-off-by: Asias He <asias at redhat.com>
> > > ---
> > 
> > Everything looks fine after initial testing, so also applied to
> > target-pending/for-next.
> > 
> > Nice optimization.  :)
> > 
> > --nab
> 
> Maybe best if you put the llist optimization on that branch too then?
> 
<nod>, already applied that one ahead of this series.

--nab