Xen devel - Feb 2013 - [PATCH] PVH linux: Use ballooning to allocate grant table pages

This patch fixes a fixme in Linux to use alloc_xenballooned_pages() to
allocate pfns for grant table pages instead of kmalloc. This also
simplifies add to physmap on the xen side a bit.

Signed-off-by: Mukesh Rathor <mukesh.rathor@oracle.com>

---
 drivers/xen/grant-table.c |   37 ++++++++++++++++++++++++++-----------
 1 files changed, 26 insertions(+), 11 deletions(-)

diff --git a/drivers/xen/grant-table.c b/drivers/xen/grant-table.c
index 9c0019d..d731f39 100644
--- a/drivers/xen/grant-table.c
+++ b/drivers/xen/grant-table.c
@@ -47,6 +47,7 @@
 #include <xen/grant_table.h>
 #include <xen/interface/memory.h>
 #include <xen/hvc-console.h>
+#include <xen/balloon.h>
 #include <asm/xen/hypercall.h>
 #include <asm/xen/interface.h>
 
@@ -1138,27 +1139,41 @@ static void gnttab_request_version(void)
 		grant_table_version);
 }
 
+static int xlated_setup_gnttab_pages(int numpages, void **addr)
+{
+	int i, rc;
+	unsigned long pfns[numpages];
+	struct page *pages[numpages];
+
+	rc = alloc_xenballooned_pages(numpages, pages, 0);
+	if (rc != 0) {
+		pr_warn("%s Could not balloon alloc %d pfns rc:%d\n", __func__,
+			numpages, rc);
+		return -ENOMEM;
+	}
+	for (i = 0; i < numpages; i++)
+		pfns[i] = page_to_pfn(pages[i]);
+
+	rc = arch_gnttab_map_shared(pfns, numpages, numpages, addr);
+	return rc;
+}
+
 int gnttab_resume(void)
 {
-	unsigned int max_nr_gframes;
-	char *kmsg = "Failed to kmalloc pages for pv in hvm grant frames\n";
+	unsigned int rc, max_nr_gframes;
 
 	gnttab_request_version();
 	max_nr_gframes = gnttab_max_grant_frames();
 	if (max_nr_gframes < nr_grant_frames)
 		return -ENOSYS;
 
-	/* PVH note: xen will free existing kmalloc''d mfn in
-	 * XENMEM_add_to_physmap. TBD/FIXME: use xen ballooning instead of
-	 * kmalloc(). */
 	if (xen_pv_domain() && xen_feature(XENFEAT_auto_translated_physmap)
&&
 	    !gnttab_shared.addr) {
-		gnttab_shared.addr -			kmalloc(max_nr_gframes * PAGE_SIZE, GFP_KERNEL);
-		if (!gnttab_shared.addr) {
-			pr_warn("%s", kmsg);
-			return -ENOMEM;
-		}
+
+		rc = xlated_setup_gnttab_pages(max_nr_gframes,
+					       &gnttab_shared.addr);
+		if (rc != 0)
+			return rc;
 	}
 	if (xen_pv_domain())
 		return gnttab_map(0, nr_grant_frames - 1);
-- 
1.7.2.3

On Thu, 31 Jan 2013 18:30:15 -0800
Mukesh Rathor <mukesh.rathor@oracle.com> wrote:
> This patch fixes a fixme in Linux to use alloc_xenballooned_pages() to
> allocate pfns for grant table pages instead of kmalloc. This also
> simplifies add to physmap on the xen side a bit.
Looking at it again, I realized rc should be signed in gnttab_resume().
Below again. Thanks.

Signed-off-by: Mukesh Rathor <mukesh.rathor@oracle.com>

diff --git a/drivers/xen/grant-table.c b/drivers/xen/grant-table.c
index 9c0019d..7a630bb 100644
--- a/drivers/xen/grant-table.c
+++ b/drivers/xen/grant-table.c
@@ -47,6 +47,7 @@
 #include <xen/grant_table.h>
 #include <xen/interface/memory.h>
 #include <xen/hvc-console.h>
+#include <xen/balloon.h>
 #include <asm/xen/hypercall.h>
 #include <asm/xen/interface.h>
 
@@ -1138,27 +1139,42 @@ static void gnttab_request_version(void)
 		grant_table_version);
 }
 
+static int xlated_setup_gnttab_pages(int numpages, void **addr)
+{
+	int i, rc;
+	unsigned long pfns[numpages];
+	struct page *pages[numpages];
+
+	rc = alloc_xenballooned_pages(numpages, pages, 0);
+	if (rc != 0) {
+		pr_warn("%s Could not balloon alloc %d pfns rc:%d\n", __func__,
+			numpages, rc);
+		return -ENOMEM;
+	}
+	for (i = 0; i < numpages; i++)
+		pfns[i] = page_to_pfn(pages[i]);
+
+	rc = arch_gnttab_map_shared(pfns, numpages, numpages, addr);
+	return rc;
+}
+
 int gnttab_resume(void)
 {
+	int rc;
 	unsigned int max_nr_gframes;
-	char *kmsg = "Failed to kmalloc pages for pv in hvm grant frames\n";
 
 	gnttab_request_version();
 	max_nr_gframes = gnttab_max_grant_frames();
 	if (max_nr_gframes < nr_grant_frames)
 		return -ENOSYS;
 
-	/* PVH note: xen will free existing kmalloc''d mfn in
-	 * XENMEM_add_to_physmap. TBD/FIXME: use xen ballooning instead of
-	 * kmalloc(). */
 	if (xen_pv_domain() && xen_feature(XENFEAT_auto_translated_physmap)
&&
 	    !gnttab_shared.addr) {
-		gnttab_shared.addr -			kmalloc(max_nr_gframes * PAGE_SIZE, GFP_KERNEL);
-		if (!gnttab_shared.addr) {
-			pr_warn("%s", kmsg);
-			return -ENOMEM;
-		}
+
+		rc = xlated_setup_gnttab_pages(max_nr_gframes,
+					       &gnttab_shared.addr);
+		if (rc != 0)
+			return rc;
 	}
 	if (xen_pv_domain())
 		return gnttab_map(0, nr_grant_frames - 1);

On Thu, 31 Jan 2013 18:44:46 -0800
Mukesh Rathor <mukesh.rathor@oracle.com> wrote:
> On Thu, 31 Jan 2013 18:30:15 -0800
> Mukesh Rathor <mukesh.rathor@oracle.com> wrote:
> 
> > This patch fixes a fixme in Linux to use alloc_xenballooned_pages()
> > to allocate pfns for grant table pages instead of kmalloc. This also
> > simplifies add to physmap on the xen side a bit.
> 
> Looking at it again, I realized rc should be signed in
> gnttab_resume(). Below again. Thanks.
Konrad, Please hold off on this patch. I discovered an issue on the 
domU side with this change. I''m currently investigating if
it''s
related.

thanks,
Mukesh

On Fri, 1 Feb 2013 14:00:58 -0800
Mukesh Rathor <mukesh.rathor@oracle.com> wrote:
> On Thu, 31 Jan 2013 18:44:46 -0800
> Mukesh Rathor <mukesh.rathor@oracle.com> wrote:
> 
> > On Thu, 31 Jan 2013 18:30:15 -0800
> > Mukesh Rathor <mukesh.rathor@oracle.com> wrote:
> > 
> > > This patch fixes a fixme in Linux to use
> > > alloc_xenballooned_pages() to allocate pfns for grant table pages
> > > instead of kmalloc. This also simplifies add to physmap on the
> > > xen side a bit.
> > 
> > Looking at it again, I realized rc should be signed in
> > gnttab_resume(). Below again. Thanks.
> 
> Konrad, Please hold off on this patch. I discovered an issue on the 
> domU side with this change. I''m currently investigating if
it''s
> related.
Ah right, I forgot the pfn''s from balloon may not be always contigous.
Besides these are special pfns so to speak, so in gnttab_map()
virt_to_pfn doesn''t work. 

I''m gonna have to create a separate gnttab map routine for pvh case, it
appears. Shouldn''t be too bad tho.

thanks,
Mukesh

This patch fixes a fixme in Linux to use alloc_xenballooned_pages() to
allocate pfns for grant table pages instead of kmalloc. This also
simplifies add to physmap on the xen side a bit.

Signed-off-by: Mukesh Rathor <mukesh.rathor@oracle.com>

diff --git a/drivers/xen/grant-table.c b/drivers/xen/grant-table.c
index 9c0019d..fdb1d88 100644
--- a/drivers/xen/grant-table.c
+++ b/drivers/xen/grant-table.c
@@ -47,6 +47,7 @@
 #include <xen/grant_table.h>
 #include <xen/interface/memory.h>
 #include <xen/hvc-console.h>
+#include <xen/balloon.h>
 #include <asm/xen/hypercall.h>
 #include <asm/xen/interface.h>
 
@@ -1026,10 +1027,22 @@ static void gnttab_unmap_frames_v2(void)
 	arch_gnttab_unmap(grstatus, nr_status_frames(nr_grant_frames));
 }
 
+static xen_pfn_t pvh_get_grant_pfn(int grant_idx)
+{
+	unsigned long vaddr;
+	unsigned int level;
+	pte_t *pte;
+
+	vaddr = (unsigned long)(gnttab_shared.addr) + grant_idx * PAGE_SIZE;
+	pte = lookup_address(vaddr, &level);
+	BUG_ON(pte == NULL);
+	return pte_mfn(*pte);
+}
+
 static int gnttab_map(unsigned int start_idx, unsigned int end_idx)
 {
 	struct gnttab_setup_table setup;
-	unsigned long *frames, start_gpfn;
+	unsigned long *frames, start_gpfn = 0;
 	unsigned int nr_gframes = end_idx + 1;
 	int rc;
 
@@ -1040,8 +1053,6 @@ static int gnttab_map(unsigned int start_idx, unsigned int
end_idx)
 
 		if (xen_hvm_domain())
 			start_gpfn = xen_hvm_resume_frames >> PAGE_SHIFT;
-		else
-			start_gpfn = virt_to_pfn(gnttab_shared.addr);
 		/*
 		 * Loop backwards, so that the first hypercall has the largest
 		 * index, ensuring that the table will grow only once.
@@ -1050,7 +1061,11 @@ static int gnttab_map(unsigned int start_idx, unsigned
int end_idx)
 			xatp.domid = DOMID_SELF;
 			xatp.idx = i;
 			xatp.space = XENMAPSPACE_grant_table;
-			xatp.gpfn = start_gpfn + i;
+			if (xen_hvm_domain())
+				xatp.gpfn = start_gpfn + i;
+			else
+				xatp.gpfn = pvh_get_grant_pfn(i);
+
 			rc = HYPERVISOR_memory_op(XENMEM_add_to_physmap, &xatp);
 			if (rc != 0) {
 				printk(KERN_WARNING
@@ -1138,27 +1153,51 @@ static void gnttab_request_version(void)
 		grant_table_version);
 }
 
+/*
+ * PVH: we need three things: virtual address, pfns, and mfns. The pfns
+ * are allocated via ballooning, then we call arch_gnttab_map_shared to
+ * allocate the VA and put pfn''s in the pte''s for the VA. The
mfn''s are
+ * finally allocated in gnttab_map() by xen which also populates the P2M.
+ */
+static int xlated_setup_gnttab_pages(unsigned long numpages, void **addr)
+{
+	int i, rc;
+	unsigned long pfns[numpages];
+	struct page *pages[numpages];
+
+	rc = alloc_xenballooned_pages(numpages, pages, 0);
+	if (rc != 0) {
+		pr_warn("%s Couldn''t balloon alloc %ld pfns rc:%d\n",
__func__,
+			numpages, rc);
+		return rc;
+	}
+	for (i = 0; i < numpages; i++)
+		pfns[i] = page_to_pfn(pages[i]);
+
+	rc = arch_gnttab_map_shared(pfns, numpages, numpages, addr);
+	if (rc != 0)
+		free_xenballooned_pages(numpages, pages);
+
+	return rc;
+}
+
 int gnttab_resume(void)
 {
+	int rc;
 	unsigned int max_nr_gframes;
-	char *kmsg = "Failed to kmalloc pages for pv in hvm grant frames\n";
 
 	gnttab_request_version();
 	max_nr_gframes = gnttab_max_grant_frames();
 	if (max_nr_gframes < nr_grant_frames)
 		return -ENOSYS;
 
-	/* PVH note: xen will free existing kmalloc''d mfn in
-	 * XENMEM_add_to_physmap. TBD/FIXME: use xen ballooning instead of
-	 * kmalloc(). */
 	if (xen_pv_domain() && xen_feature(XENFEAT_auto_translated_physmap)
&&
 	    !gnttab_shared.addr) {
-		gnttab_shared.addr -			kmalloc(max_nr_gframes * PAGE_SIZE, GFP_KERNEL);
-		if (!gnttab_shared.addr) {
-			pr_warn("%s", kmsg);
-			return -ENOMEM;
-		}
+
+		rc = xlated_setup_gnttab_pages((unsigned long)max_nr_gframes,
+					       &gnttab_shared.addr);
+		if (rc != 0)
+			return rc;
 	}
 	if (xen_pv_domain())
 		return gnttab_map(0, nr_grant_frames - 1);

On Thu, Jan 31, 2013 at 06:30:15PM -0800, Mukesh Rathor
wrote:
> This patch fixes a fixme in Linux to use alloc_xenballooned_pages() to
> allocate pfns for grant table pages instead of kmalloc. This also
> simplifies add to physmap on the xen side a bit.
Pulled this.
> 
> Signed-off-by: Mukesh Rathor <mukesh.rathor@oracle.com>
> 
> ---
>  drivers/xen/grant-table.c |   37 ++++++++++++++++++++++++++-----------
>  1 files changed, 26 insertions(+), 11 deletions(-)
> 
> diff --git a/drivers/xen/grant-table.c b/drivers/xen/grant-table.c
> index 9c0019d..d731f39 100644
> --- a/drivers/xen/grant-table.c
> +++ b/drivers/xen/grant-table.c
> @@ -47,6 +47,7 @@
>  #include <xen/grant_table.h>
>  #include <xen/interface/memory.h>
>  #include <xen/hvc-console.h>
> +#include <xen/balloon.h>
>  #include <asm/xen/hypercall.h>
>  #include <asm/xen/interface.h>
>  
> @@ -1138,27 +1139,41 @@ static void gnttab_request_version(void)
>  		grant_table_version);
>  }
>  
> +static int xlated_setup_gnttab_pages(int numpages, void **addr)
> +{
> +	int i, rc;
> +	unsigned long pfns[numpages];
> +	struct page *pages[numpages];
> +
> +	rc = alloc_xenballooned_pages(numpages, pages, 0);
> +	if (rc != 0) {
> +		pr_warn("%s Could not balloon alloc %d pfns rc:%d\n",
__func__,
> +			numpages, rc);
> +		return -ENOMEM;
> +	}
> +	for (i = 0; i < numpages; i++)
> +		pfns[i] = page_to_pfn(pages[i]);
> +
> +	rc = arch_gnttab_map_shared(pfns, numpages, numpages, addr);
> +	return rc;
> +}
> +
>  int gnttab_resume(void)
>  {
> -	unsigned int max_nr_gframes;
> -	char *kmsg = "Failed to kmalloc pages for pv in hvm grant
frames\n";
> +	unsigned int rc, max_nr_gframes;
>  
>  	gnttab_request_version();
>  	max_nr_gframes = gnttab_max_grant_frames();
>  	if (max_nr_gframes < nr_grant_frames)
>  		return -ENOSYS;
>  
> -	/* PVH note: xen will free existing kmalloc''d mfn in
> -	 * XENMEM_add_to_physmap. TBD/FIXME: use xen ballooning instead of
> -	 * kmalloc(). */
>  	if (xen_pv_domain() &&
xen_feature(XENFEAT_auto_translated_physmap) &&
>  	    !gnttab_shared.addr) {
> -		gnttab_shared.addr > -			kmalloc(max_nr_gframes * PAGE_SIZE,
GFP_KERNEL);
> -		if (!gnttab_shared.addr) {
> -			pr_warn("%s", kmsg);
> -			return -ENOMEM;
> -		}
> +
> +		rc = xlated_setup_gnttab_pages(max_nr_gframes,
> +					       &gnttab_shared.addr);
> +		if (rc != 0)
> +			return rc;
>  	}
>  	if (xen_pv_domain())
>  		return gnttab_map(0, nr_grant_frames - 1);
> -- 
> 1.7.2.3
>

On Wed, 6 Feb 2013 10:49:13 -0500
Konrad Rzeszutek Wilk <konrad@darnok.org> wrote:
> On Thu, Jan 31, 2013 at 06:30:15PM -0800, Mukesh Rathor wrote:
> > This patch fixes a fixme in Linux to use alloc_xenballooned_pages()
> > to allocate pfns for grant table pages instead of kmalloc. This also
> > simplifies add to physmap on the xen side a bit.
> 
> Pulled this.
> > 

Konrad, no, there was a follow up email on this thread to discard this.
Please discard this. I resent yesterday with proper fixes. I realize
now I should''ve given one yesterday version number. My bad, this head 
cold is crippling my brain :).. 

Sorry for the confusion.

Mukesh

Following is the latest patch I emailed yesterday :


This patch fixes a fixme in Linux to use alloc_xenballooned_pages() to
allocate pfns for grant table pages instead of kmalloc. This also
simplifies add to physmap on the xen side a bit.

Signed-off-by: Mukesh Rathor <mukesh.rathor@oracle.com>

diff --git a/drivers/xen/grant-table.c b/drivers/xen/grant-table.c
index 9c0019d..fdb1d88 100644
--- a/drivers/xen/grant-table.c
+++ b/drivers/xen/grant-table.c
@@ -47,6 +47,7 @@
 #include <xen/grant_table.h>
 #include <xen/interface/memory.h>
 #include <xen/hvc-console.h>
+#include <xen/balloon.h>
 #include <asm/xen/hypercall.h>
 #include <asm/xen/interface.h>
 
@@ -1026,10 +1027,22 @@ static void gnttab_unmap_frames_v2(void)
 	arch_gnttab_unmap(grstatus, nr_status_frames(nr_grant_frames));
 }
 
+static xen_pfn_t pvh_get_grant_pfn(int grant_idx)
+{
+	unsigned long vaddr;
+	unsigned int level;
+	pte_t *pte;
+
+	vaddr = (unsigned long)(gnttab_shared.addr) + grant_idx * PAGE_SIZE;
+	pte = lookup_address(vaddr, &level);
+	BUG_ON(pte == NULL);
+	return pte_mfn(*pte);
+}
+
 static int gnttab_map(unsigned int start_idx, unsigned int end_idx)
 {
 	struct gnttab_setup_table setup;
-	unsigned long *frames, start_gpfn;
+	unsigned long *frames, start_gpfn = 0;
 	unsigned int nr_gframes = end_idx + 1;
 	int rc;
 
@@ -1040,8 +1053,6 @@ static int gnttab_map(unsigned int start_idx, unsigned int
end_idx)
 
 		if (xen_hvm_domain())
 			start_gpfn = xen_hvm_resume_frames >> PAGE_SHIFT;
-		else
-			start_gpfn = virt_to_pfn(gnttab_shared.addr);
 		/*
 		 * Loop backwards, so that the first hypercall has the largest
 		 * index, ensuring that the table will grow only once.
@@ -1050,7 +1061,11 @@ static int gnttab_map(unsigned int start_idx, unsigned
int end_idx)
 			xatp.domid = DOMID_SELF;
 			xatp.idx = i;
 			xatp.space = XENMAPSPACE_grant_table;
-			xatp.gpfn = start_gpfn + i;
+			if (xen_hvm_domain())
+				xatp.gpfn = start_gpfn + i;
+			else
+				xatp.gpfn = pvh_get_grant_pfn(i);
+
 			rc = HYPERVISOR_memory_op(XENMEM_add_to_physmap, &xatp);
 			if (rc != 0) {
 				printk(KERN_WARNING
@@ -1138,27 +1153,51 @@ static void gnttab_request_version(void)
 		grant_table_version);
 }
 
+/*
+ * PVH: we need three things: virtual address, pfns, and mfns. The pfns
+ * are allocated via ballooning, then we call arch_gnttab_map_shared to
+ * allocate the VA and put pfn''s in the pte''s for the VA. The
mfn''s are
+ * finally allocated in gnttab_map() by xen which also populates the P2M.
+ */
+static int xlated_setup_gnttab_pages(unsigned long numpages, void **addr)
+{
+	int i, rc;
+	unsigned long pfns[numpages];
+	struct page *pages[numpages];
+
+	rc = alloc_xenballooned_pages(numpages, pages, 0);
+	if (rc != 0) {
+		pr_warn("%s Couldn''t balloon alloc %ld pfns rc:%d\n",
__func__,
+			numpages, rc);
+		return rc;
+	}
+	for (i = 0; i < numpages; i++)
+		pfns[i] = page_to_pfn(pages[i]);
+
+	rc = arch_gnttab_map_shared(pfns, numpages, numpages, addr);
+	if (rc != 0)
+		free_xenballooned_pages(numpages, pages);
+
+	return rc;
+}
+
 int gnttab_resume(void)
 {
+	int rc;
 	unsigned int max_nr_gframes;
-	char *kmsg = "Failed to kmalloc pages for pv in hvm grant frames\n";
 
 	gnttab_request_version();
 	max_nr_gframes = gnttab_max_grant_frames();
 	if (max_nr_gframes < nr_grant_frames)
 		return -ENOSYS;
 
-	/* PVH note: xen will free existing kmalloc''d mfn in
-	 * XENMEM_add_to_physmap. TBD/FIXME: use xen ballooning instead of
-	 * kmalloc(). */
 	if (xen_pv_domain() && xen_feature(XENFEAT_auto_translated_physmap)
&&
 	    !gnttab_shared.addr) {
-		gnttab_shared.addr -			kmalloc(max_nr_gframes * PAGE_SIZE, GFP_KERNEL);
-		if (!gnttab_shared.addr) {
-			pr_warn("%s", kmsg);
-			return -ENOMEM;
-		}
+
+		rc = xlated_setup_gnttab_pages((unsigned long)max_nr_gframes,
+					       &gnttab_shared.addr);
+		if (rc != 0)
+			return rc;
 	}
 	if (xen_pv_domain())
 		return gnttab_map(0, nr_grant_frames - 1);