Xen devel - Aug 2009 - [PATCH][v2.6.29][XEN] Return unused memory to hypervisor

Hi,

when running linux as XEN guest and use boot parameter mem= to set memory lower
then is assigned to guest, not used memory should be returned to hypervisor as
free. This is working with kernel available on xen.org pages, but is not working
with kernel 2.6.29. Comparing both kernels I found code for returning unused
memory to hypervisor is missing. Following patch add this functionality to
2.6.29 kernel.

Miroslav Rezanina <mrezanin@redhat.com>
--
diff --git a/arch/x86/kernel/setup.c b/arch/x86/kernel/setup.c
index 6a8811a..fd6b0e7 100644
--- a/arch/x86/kernel/setup.c
+++ b/arch/x86/kernel/setup.c
@@ -118,6 +118,10 @@ struct boot_params __initdata boot_params;
 struct boot_params boot_params;
 #endif
 
+#ifdef CONFIG_XEN
+void __init xen_return_unused_mem(void);
+#endif
+
 /*
  * Machine setup..
  */
@@ -920,6 +924,9 @@ void __init setup_arch(char **cmdline_p)
 	paging_init();
 	paravirt_pagetable_setup_done(swapper_pg_dir);
 	paravirt_post_allocator_init();
+#ifdef CONFIG_XEN
+	xen_return_unused_mem();
+#endif
 
 #ifdef CONFIG_X86_64
 	map_vsyscall();
diff --git a/arch/x86/xen/setup.c b/arch/x86/xen/setup.c
index 15c6c68..bc5d2bc 100644
--- a/arch/x86/xen/setup.c
+++ b/arch/x86/xen/setup.c
@@ -20,6 +20,7 @@
 #include <xen/page.h>
 #include <xen/interface/callback.h>
 #include <xen/interface/physdev.h>
+#include <xen/interface/memory.h>
 #include <xen/features.h>
 
 #include "xen-ops.h"
@@ -34,6 +35,36 @@ extern void xen_syscall32_target(void);
 
 
 /**
+ * Author: Miroslav Rezanina <mrezanin@redhat.com>
+ * Function retuns unused memory to hypevisor
+ **/
+void __init xen_return_unused_mem(void)
+{
+	if (xen_start_info->nr_pages > max_pfn) {
+        /*
+	* the max_pfn was shrunk (probably by mem+	* kernel parameter); shrink
reservation with the HV
+	*/
+	struct xen_memory_reservation reservation = {
+	.address_bits = 0,
+	.extent_order = 0,
+	.domid = DOMID_SELF
+	};
+	unsigned int difference;
+	int ret;
+
+	difference = xen_start_info->nr_pages - max_pfn;
+
+	set_xen_guest_handle(reservation.extent_start,
+		((unsigned long *)xen_start_info->mfn_list) + max_pfn);
+	reservation.nr_extents = difference;
+	ret = HYPERVISOR_memory_op(XENMEM_decrease_reservation,
+		&reservation);
+	BUG_ON (ret != difference);
+	}
+}
+
+/**
  * machine_specific_memory_setup - Hook for machine specific memory setup.
  **/
 

_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xensource.com
http://lists.xensource.com/xen-devel

On 08/19/09 06:05, Miroslav Rezanina wrote:
> when running linux as XEN guest and use boot parameter mem= to set memory
lower then is assigned to guest, not used memory should be returned to
hypervisor as free. This is working with kernel available on xen.org pages, but
is not working with kernel 2.6.29. Comparing both kernels I found code for
returning unused memory to hypervisor is missing. Following patch add this
functionality to 2.6.29 kernel.
>   
The idea is sound, but I think it might be better to walk the e820
table, and remove any memory ranges which aren''t marked as E820_RAM. 
That makes it possible to carve holes in the address space as well as
simply truncate it.

Also, something appears to have smashed your indentation.

    J
> Miroslav Rezanina <mrezanin@redhat.com>
> --
> diff --git a/arch/x86/kernel/setup.c b/arch/x86/kernel/setup.c
> index 6a8811a..fd6b0e7 100644
> --- a/arch/x86/kernel/setup.c
> +++ b/arch/x86/kernel/setup.c
> @@ -118,6 +118,10 @@ struct boot_params __initdata boot_params;
>  struct boot_params boot_params;
>  #endif
>  
> +#ifdef CONFIG_XEN
> +void __init xen_return_unused_mem(void);
> +#endif
> +
>  /*
>   * Machine setup..
>   */
> @@ -920,6 +924,9 @@ void __init setup_arch(char **cmdline_p)
>  	paging_init();
>  	paravirt_pagetable_setup_done(swapper_pg_dir);
>  	paravirt_post_allocator_init();
> +#ifdef CONFIG_XEN
> +	xen_return_unused_mem();
> +#endif
>  
>  #ifdef CONFIG_X86_64
>  	map_vsyscall();
> diff --git a/arch/x86/xen/setup.c b/arch/x86/xen/setup.c
> index 15c6c68..bc5d2bc 100644
> --- a/arch/x86/xen/setup.c
> +++ b/arch/x86/xen/setup.c
> @@ -20,6 +20,7 @@
>  #include <xen/page.h>
>  #include <xen/interface/callback.h>
>  #include <xen/interface/physdev.h>
> +#include <xen/interface/memory.h>
>  #include <xen/features.h>
>  
>  #include "xen-ops.h"
> @@ -34,6 +35,36 @@ extern void xen_syscall32_target(void);
>  
>  
>  /**
> + * Author: Miroslav Rezanina <mrezanin@redhat.com>
> + * Function retuns unused memory to hypevisor
> + **/
> +void __init xen_return_unused_mem(void)
> +{
> +	if (xen_start_info->nr_pages > max_pfn) {
> +        /*
> +	* the max_pfn was shrunk (probably by mem> +	* kernel parameter);
shrink reservation with the HV
> +	*/
> +	struct xen_memory_reservation reservation = {
> +	.address_bits = 0,
> +	.extent_order = 0,
> +	.domid = DOMID_SELF
> +	};
> +	unsigned int difference;
> +	int ret;
> +
> +	difference = xen_start_info->nr_pages - max_pfn;
> +
> +	set_xen_guest_handle(reservation.extent_start,
> +		((unsigned long *)xen_start_info->mfn_list) + max_pfn);
> +	reservation.nr_extents = difference;
> +	ret = HYPERVISOR_memory_op(XENMEM_decrease_reservation,
> +		&reservation);
> +	BUG_ON (ret != difference);
> +	}
> +}
> +
> +/**
>   * machine_specific_memory_setup - Hook for machine specific memory setup.
>   **/
>  
>
>   

_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xensource.com
http://lists.xensource.com/xen-devel

----- Original Message -----
From: "Jeremy Fitzhardinge" <jeremy@goop.org>
To: "Miroslav Rezanina" <mrezanin@redhat.com>
Cc: linux-kernel@vger.kernel.org, xen-devel@lists.xensource.com, "Gianluca
Guida" <gianluca.guida@citrix.com>
Sent: Wednesday, August 19, 2009 6:16:33 PM GMT +01:00 Amsterdam / Berlin / Bern
/ Rome / Stockholm / Vienna
Subject: Re: [PATCH][v2.6.29][XEN] Return unused memory to hypervisor
>>On 08/19/09 06:05, Miroslav Rezanina wrote:
>> when running linux as XEN guest and use boot parameter mem= to set
>> memory lower then is assigned to guest, not used memory should be 
>> returned to hypervisor as free. This is working with kernel 
>> available on xen.org pages, but is not working with kernel 2.6.29.
>> Comparing both kernels I found code for returning unused memory to 
>> hypervisor is missing. Following patch add this functionality to 
>>2.6.29 kernel.
>>  
>
> The idea is sound, but I think it might be better to walk the e820
> table, and remove any memory ranges which aren''t marked as
E820_RAM.
> That makes it possible to carve holes in the address space as well as
> simply truncate it.
Hi Jeremy,
there is handled e820 map in guest. However, this patch informs 
hypervisor, that guest uses less memory than was assigned to it. 
If hypervisor is not informed, memory is reserved for guest that
do not need it. If hypervisor is informed, he decrease memory
reservation for guest and unused memory is marked as free 
for use by other guests.

Mirek
> Also, something appears to have smashed your indentation.
>
>    J
Oh, something goes wrong. Resending patch:

diff --git a/arch/x86/kernel/setup.c b/arch/x86/kernel/setup.c
index 6a8811a..fd6b0e7 100644
--- a/arch/x86/kernel/setup.c
+++ b/arch/x86/kernel/setup.c
@@ -118,6 +118,10 @@ struct boot_params __initdata boot_params;
 struct boot_params boot_params;
 #endif
 
+#ifdef CONFIG_XEN
+void __init xen_return_unused_mem(void);
+#endif
+
 /*
  * Machine setup..
  */
@@ -920,6 +924,9 @@ void __init setup_arch(char **cmdline_p)
 	paging_init();
 	paravirt_pagetable_setup_done(swapper_pg_dir);
 	paravirt_post_allocator_init();
+#ifdef CONFIG_XEN
+	xen_return_unused_mem();
+#endif
 
 #ifdef CONFIG_X86_64
 	map_vsyscall();
diff --git a/arch/x86/xen/setup.c b/arch/x86/xen/setup.c
index 15c6c68..bc5d2bc 100644
--- a/arch/x86/xen/setup.c
+++ b/arch/x86/xen/setup.c
@@ -20,6 +20,7 @@
 #include <xen/page.h>
 #include <xen/interface/callback.h>
 #include <xen/interface/physdev.h>
+#include <xen/interface/memory.h>
 #include <xen/features.h>
 
 #include "xen-ops.h"
@@ -34,6 +35,36 @@ extern void xen_syscall32_target(void);
 
 
 /**
+ * Author: Miroslav Rezanina <mrezanin@redhat.com>
+ * Function retuns unused memory to hypevisor
+ **/
+void __init xen_return_unused_mem(void)
+{
+	if (xen_start_info->nr_pages > max_pfn) {
+        /*
+	* the max_pfn was shrunk (probably by mem+	* kernel parameter); shrink
reservation with the HV
+	*/
+	struct xen_memory_reservation reservation = {
+	.address_bits = 0,
+	.extent_order = 0,
+	.domid = DOMID_SELF
+	};
+	unsigned int difference;
+	int ret;
+
+	difference = xen_start_info->nr_pages - max_pfn;
+
+	set_xen_guest_handle(reservation.extent_start,
+		((unsigned long *)xen_start_info->mfn_list) + max_pfn);
+	reservation.nr_extents = difference;
+	ret = HYPERVISOR_memory_op(XENMEM_decrease_reservation,
+		&reservation);
+	BUG_ON (ret != difference);
+	}
+}
+
+/**
  * machine_specific_memory_setup - Hook for machine specific memory setup.
  **/
 

_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xensource.com
http://lists.xensource.com/xen-devel

Miroslav Rezanina writes:
 > Hi,
 > 
 > when running linux as XEN guest and use boot parameter mem= to set memory
lower then is assigned to guest, not used memory should be returned to
hypervisor as free. This is working with kernel available on xen.org pages, but
is not working with kernel 2.6.29. Comparing both kernels I found code for
returning unused memory to hypervisor is missing. Following patch add this
functionality to 2.6.29 kernel.
 > 

A good idea would be to avoid putting this code in the generic kernel
code. For now just placing it in at the end of Xen''s post-allocator
init would make it completely transparent to the non-xen kernel.

There''s a patch in Jeremy''s rebase/master (at the moment
reverted)
that allows guest to boot ballooned (which is the opposite of this
case, roughly) which does this kind of calculations while walking the
e820 table. Perhaps this can be moved there after things get to work
again.

Thanks,
Gianluca

 > Miroslav Rezanina <mrezanin@redhat.com>
 > --
 > diff --git a/arch/x86/kernel/setup.c b/arch/x86/kernel/setup.c
 > index 6a8811a..fd6b0e7 100644
 > --- a/arch/x86/kernel/setup.c
 > +++ b/arch/x86/kernel/setup.c
 > @@ -118,6 +118,10 @@ struct boot_params __initdata boot_params;
 >  struct boot_params boot_params;
 >  #endif
 >  
 > +#ifdef CONFIG_XEN
 > +void __init xen_return_unused_mem(void);
 > +#endif
 > +
 >  /*
 >   * Machine setup..
 >   */
 > @@ -920,6 +924,9 @@ void __init setup_arch(char **cmdline_p)
 >  	paging_init();
 >  	paravirt_pagetable_setup_done(swapper_pg_dir);
 >  	paravirt_post_allocator_init();
 > +#ifdef CONFIG_XEN
 > +	xen_return_unused_mem();
 > +#endif
 >  
 >  #ifdef CONFIG_X86_64
 >  	map_vsyscall();
 > diff --git a/arch/x86/xen/setup.c b/arch/x86/xen/setup.c
 > index 15c6c68..bc5d2bc 100644
 > --- a/arch/x86/xen/setup.c
 > +++ b/arch/x86/xen/setup.c
 > @@ -20,6 +20,7 @@
 >  #include <xen/page.h>
 >  #include <xen/interface/callback.h>
 >  #include <xen/interface/physdev.h>
 > +#include <xen/interface/memory.h>
 >  #include <xen/features.h>
 >  
 >  #include "xen-ops.h"
 > @@ -34,6 +35,36 @@ extern void xen_syscall32_target(void);
 >  
 >  
 >  /**
 > + * Author: Miroslav Rezanina <mrezanin@redhat.com>
 > + * Function retuns unused memory to hypevisor
 > + **/
 > +void __init xen_return_unused_mem(void)
 > +{
 > +	if (xen_start_info->nr_pages > max_pfn) {
 > +        /*
 > +	* the max_pfn was shrunk (probably by mem > +	* kernel parameter);
shrink reservation with the HV
 > +	*/
 > +	struct xen_memory_reservation reservation = {
 > +	.address_bits = 0,
 > +	.extent_order = 0,
 > +	.domid = DOMID_SELF
 > +	};
 > +	unsigned int difference;
 > +	int ret;
 > +
 > +	difference = xen_start_info->nr_pages - max_pfn;
 > +
 > +	set_xen_guest_handle(reservation.extent_start,
 > +		((unsigned long *)xen_start_info->mfn_list) + max_pfn);
 > +	reservation.nr_extents = difference;
 > +	ret = HYPERVISOR_memory_op(XENMEM_decrease_reservation,
 > +		&reservation);
 > +	BUG_ON (ret != difference);
 > +	}
 > +}
 > +
 > +/**
 >   * machine_specific_memory_setup - Hook for machine specific memory
setup.
 >   **/
 >  
 > 
 > _______________________________________________
 > Xen-devel mailing list
 > Xen-devel@lists.xensource.com
 > http://lists.xensource.com/xen-devel

_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xensource.com
http://lists.xensource.com/xen-devel

----- Original Message -----
From: "Gianluca Guida" <gianluca.guida@citrix.com>
To: "Miroslav Rezanina" <mrezanin@redhat.com>
Cc: jeremy@goop.org, xen-devel@lists.xensource.com, linux-kernel@vger.kernel.org
Sent: Thursday, August 20, 2009 11:31:34 AM GMT +01:00 Amsterdam / Berlin / Bern
/ Rome / Stockholm / Vienna
Subject: [Xen-devel] [PATCH][v2.6.29][XEN] Return unused memory to hypervisor

Miroslav Rezanina writes:
> > Hi,
> > 
> > when running linux as XEN guest and use boot parameter mem= to set
memory lower then is assigned to guest, not used memory should be returned to
hypervisor as free. This is working with kernel available on xen.org pages, but
is not working with kernel 2.6.29. Comparing both kernels I found code for
returning unused memory to hypervisor is missing. Following patch add this
functionality to 2.6.29 kernel.
> > 
>
> A good idea would be to avoid putting this code in the generic kernel
> code. For now just placing it in at the end of Xen''s
post-allocator
> init would make it completely transparent to the non-xen kernel.
>
Hi Gianluca,
good point. You''re right. I moved calling to xen_post_allocator_init
function.
It is better place for it.

Regards,
Mirek

----diff --git a/arch/x86/xen/enlighten.c b/arch/x86/xen/enlighten.c
index b58e963..2a9cc80 100644
--- a/arch/x86/xen/enlighten.c
+++ b/arch/x86/xen/enlighten.c
@@ -28,6 +28,7 @@
 #include <linux/console.h>
 
 #include <xen/interface/xen.h>
+#include <xen/interface/memory.h>
 #include <xen/interface/version.h>
 #include <xen/interface/physdev.h>
 #include <xen/interface/vcpu.h>
@@ -122,6 +123,36 @@ static int have_vcpu_info_placement  #endif
 	;
 
+/**
+ *  * Author: Miroslav Rezanina <mrezanin@redhat.com>
+ *   * Function retuns unused memory to hypevisor
+ *    **/
+void __init xen_return_unused_mem(void)
+{
+	if (xen_start_info->nr_pages > max_pfn) {
+	/*
+	 * the max_pfn was shrunk (probably by mem+         * kernel parameter);
shrink reservation with the HV
+	 */
+		struct xen_memory_reservation reservation = {
+			.address_bits = 0,
+			.extent_order = 0,
+			.domid = DOMID_SELF
+		};
+		unsigned int difference;
+		int ret;
+
+		difference = xen_start_info->nr_pages - max_pfn;
+
+		set_xen_guest_handle(reservation.extent_start,
+			((unsigned long *)xen_start_info->mfn_list) + max_pfn);
+		reservation.nr_extents = difference;
+		ret = HYPERVISOR_memory_op(XENMEM_decrease_reservation,
+			&reservation);
+		BUG_ON (ret != difference);
+	}
+}
+
 
 static void xen_vcpu_setup(int cpu)
 {
@@ -1057,6 +1088,8 @@ static __init void xen_post_allocator_init(void)
 	SetPagePinned(virt_to_page(level3_user_vsyscall));
 #endif
 	xen_mark_init_mm_pinned();
+
+	xen_return_unused_mem();
 }
 
 /* This is called once we have the cpu_possible_map */

_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xensource.com
http://lists.xensource.com/xen-devel

On 08/20/09 00:47, Miroslav Rezanina wrote:
> there is handled e820 map in guest. However, this patch informs 
> hypervisor, that guest uses less memory than was assigned to it. 
> If hypervisor is not informed, memory is reserved for guest that
> do not need it. If hypervisor is informed, he decrease memory
> reservation for guest and unused memory is marked as free 
> for use by other guests.
>   
Yes.  But the guest will modify its own e820 map for a number of
reasons; for example: reducing its own memory, or clearing a space for
the PCI hole.  In general we want to free any underlying pages which
don''t correspond to E820_RAM regions.

    J

_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xensource.com
http://lists.xensource.com/xen-devel

----- Original Message -----
From: "Jeremy Fitzhardinge" <jeremy@goop.org>
To: "Miroslav Rezanina" <mrezanin@redhat.com>
Cc: linux-kernel@vger.kernel.org, xen-devel@lists.xensource.com, "Gianluca
Guida" <gianluca.guida@citrix.com>
Sent: Thursday, August 20, 2009 6:39:02 PM GMT +01:00 Amsterdam / Berlin / Bern
/ Rome / Stockholm / Vienna
Subject: Re: [PATCH][v2.6.29][XEN] Return unused memory to hypervisor

On 08/20/09 00:47, Miroslav Rezanina wrote:
>> there is handled e820 map in guest. However, this patch informs 
>> hypervisor, that guest uses less memory than was assigned to it. 
>> If hypervisor is not informed, memory is reserved for guest that
>> do not need it. If hypervisor is informed, he decrease memory
>> reservation for guest and unused memory is marked as free 
>> for use by other guests.
>>   
>
> Yes.  But the guest will modify its own e820 map for a number of
> reasons; for example: reducing its own memory, or clearing a space for
> the PCI hole.  In general we want to free any underlying pages which
> don''t correspond to E820_RAM regions.
>
>    J
I agree. However, I''m not sure if xen supports such a precise handling.
  
-- 
Miroslav Rezanina
Software Engineer - Virtualization Team - XEN kernel


_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xensource.com
http://lists.xensource.com/xen-devel

Miroslav Rezanina writes:
 > ----- Original Message -----
 > From: "Jeremy Fitzhardinge" <jeremy@goop.org>
 > To: "Miroslav Rezanina" <mrezanin@redhat.com>
 > Cc: linux-kernel@vger.kernel.org, xen-devel@lists.xensource.com,
"Gianluca Guida" <gianluca.guida@citrix.com>
 > Sent: Thursday, August 20, 2009 6:39:02 PM GMT +01:00 Amsterdam / Berlin /
Bern / Rome / Stockholm / Vienna
 > Subject: Re: [PATCH][v2.6.29][XEN] Return unused memory to hypervisor
 > 
 > On 08/20/09 00:47, Miroslav Rezanina wrote:
 > >> there is handled e820 map in guest. However, this patch informs 
 > >> hypervisor, that guest uses less memory than was assigned to it. 
 > >> If hypervisor is not informed, memory is reserved for guest that
 > >> do not need it. If hypervisor is informed, he decrease memory
 > >> reservation for guest and unused memory is marked as free 
 > >> for use by other guests.
 > >>   
 > >
 > > Yes.  But the guest will modify its own e820 map for a number of
 > > reasons; for example: reducing its own memory, or clearing a space
for
 > > the PCI hole.  In general we want to free any underlying pages which
 > > don''t correspond to E820_RAM regions.
 > >
 > >    J
 > 
 > I agree. However, I''m not sure if xen supports such a precise
handling.

Xen will just get a list of mfn to remove from the domain. That is
precise enough. The scenario Jeremy is describing does happen,
especially in dom0 kernels.

G.

_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xensource.com
http://lists.xensource.com/xen-devel

On 08/19/09 06:05, Miroslav Rezanina wrote:
> when running linux as XEN guest and use boot parameter mem= to set memory
lower then is assigned to guest, not used memory should be returned to
hypervisor as free. This is working with kernel available on xen.org pages, but
is not working with kernel 2.6.29. Comparing both kernels I found code for
returning unused memory to hypervisor is missing. Following patch add this
functionality to 2.6.29 kernel.
>   
Are you planning on submitting a revised patch along the lines I suggested?

Thanks,
    J

_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xensource.com
http://lists.xensource.com/xen-devel

----- "Jeremy Fitzhardinge" <jeremy@goop.org> wrote:
> From: "Jeremy Fitzhardinge" <jeremy@goop.org>
> To: "Miroslav Rezanina" <mrezanin@redhat.com>
> Cc: linux-kernel@vger.kernel.org, xen-devel@lists.xensource.com
> Sent: Friday, September 4, 2009 1:26:24 AM GMT +01:00 Amsterdam / Berlin /
Bern / Rome / Stockholm / Vienna
> Subject: Re: [PATCH][v2.6.29][XEN] Return unused memory to hypervisor
>
> On 08/19/09 06:05, Miroslav Rezanina wrote:
> > when running linux as XEN guest and use boot parameter mem= to set
> memory lower then is assigned to guest, not used memory should be
> returned to hypervisor as free. This is working with kernel available
> on xen.org pages, but is not working with kernel 2.6.29. Comparing
> both kernels I found code for returning unused memory to hypervisor is
> missing. Following patch add this functionality to 2.6.29 kernel.
> >   
> 
> Are you planning on submitting a revised patch along the lines I
> suggested?
> 
> Thanks,
>     J
Hi Jeremy,
yes, I''m planning so. Unfortunatelly, I had some more important tasks
to do
so I was not able to rewrite patch as you suggested.

Mirek
> --
> To unsubscribe from this list: send the line "unsubscribe
> linux-kernel" in
> the body of a message to majordomo@vger.kernel.org
> More majordomo info at  http://vger.kernel.org/majordomo-info.html
> Please read the FAQ at  http://www.tux.org/lkml/
-- 
Miroslav Rezanina
Software Engineer - Virtualization Team - XEN kernel


_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xensource.com
http://lists.xensource.com/xen-devel

----- "Jeremy Fitzhardinge" <jeremy@goop.org> wrote:
> From: "Jeremy Fitzhardinge" <jeremy@goop.org>
> To: "Miroslav Rezanina" <mrezanin@redhat.com>
> Cc: linux-kernel@vger.kernel.org, xen-devel@lists.xensource.com,
"Gianluca Guida" <gianluca.guida@citrix.com>
> Sent: Thursday, August 20, 2009 6:39:02 PM GMT +01:00 Amsterdam / Berlin /
Bern / Rome / Stockholm / Vienna
> Subject: Re: [PATCH][v2.6.29][XEN] Return unused memory to hypervisor
>
> On 08/20/09 00:47, Miroslav Rezanina wrote:
> > there is handled e820 map in guest. However, this patch informs 
> > hypervisor, that guest uses less memory than was assigned to it. 
> > If hypervisor is not informed, memory is reserved for guest that
> > do not need it. If hypervisor is informed, he decrease memory
> > reservation for guest and unused memory is marked as free 
> > for use by other guests.
> >   
> 
> Yes.  But the guest will modify its own e820 map for a number of
> reasons; for example: reducing its own memory, or clearing a space
> for
> the PCI hole.  In general we want to free any underlying pages which
> don''t correspond to E820_RAM regions.
> 
>     J
> --
Hi Jeremy,
can you give me a practical example, where e820 map can have "hole"
inside,
i.e. there will be block of memory not listed in e820 map that have listed 
memory before and after it?
As I checked the source code, there is always removed memory from some point
till end of map, not from one adress till another. And I can''t image
how would
be such a case handled. Of course, there can be some special regions, as the PCI
hole, but these are marked as "Reserved".
There can be "reserved and returned" some inside memory, but this is
already
handled  by balloon driver. My patch returns memory that this driver
can''t use.
Regards,
Mirek
-- 
Miroslav Rezanina
Software Engineer - Virtualization Team - XEN kernel


_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xensource.com
http://lists.xensource.com/xen-devel

On 09/07/09 05:41, Miroslav Rezanina wrote:
> can you give me a practical example, where e820 map can have
"hole" inside,
> i.e. there will be block of memory not listed in e820 map that have listed 
> memory before and after it?
> As I checked the source code, there is always removed memory from some
point
> till end of map, not from one adress till another. And I can''t
image how would
> be such a case handled. Of course, there can be some special regions, as
the PCI
> hole, but these are marked as "Reserved".
> There can be "reserved and returned" some inside memory, but this
is already
> handled  by balloon driver. My patch returns memory that this driver
can''t use.
>   
PCI memory isn''t typically reserved; there''s just a hole in
the address
space for it.  Its up to the BIOS/OS to set the BARs for the devices
into that hole.  Reserved E820 regions are just that - reserved by the
BIOS for its own purposes.

In a Xen domain, the E820 tables are purely synthesized by the kernel''s
Xen code, and can have any content we like.  At the moment they tend to
be very simple with linear memory up to the domain''s memory size.  For
domains with PCI access - either dom0 or with pcifront - we want to be
able to carve out regions from that memory to place devices.  We may
want to manipulate the E820 tables to reshape memory for other reasons
(like pre-ballooned memory, or memory hotplug).

What I''m looking for is a nice general purpose routine which will walk
a
set of E820 entries and release back to Xen any memory which is in an
E820 hole (ie, in a gap between or after E820 entries).  That solves the
simple problem of trimming the end of the memory map, but also copes
with more complex cases that will arise.

    J

_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xensource.com
http://lists.xensource.com/xen-devel

> From: "Jeremy Fitzhardinge" <jeremy@goop.org>
> To: "Miroslav Rezanina" <mrezanin@redhat.com>
> Cc: linux-kernel@vger.kernel.org, xen-devel@lists.xensource.com
> Sent: Friday, September 4, 2009 1:26:24 AM GMT +01:00 Amsterdam / Berlin /
Bern / Rome / Stockholm / Vienna
> Subject: Re: [PATCH][v2.6.29][XEN] Return unused memory to hypervisor
>
> On 08/19/09 06:05, Miroslav Rezanina wrote:
> > when running linux as XEN guest and use boot parameter mem= to set
> memory lower then is assigned to guest, not used memory should be
> returned to hypervisor as free. This is working with kernel available
> on xen.org pages, but is not working with kernel 2.6.29. Comparing
> both kernels I found code for returning unused memory to hypervisor is
> missing. Following patch add this functionality to 2.6.29 kernel.
> >   
> 
> Are you planning on submitting a revised patch along the lines I
> suggested?
> 
> Thanks,
>     J
General version of patch. This version checks the e820 map for holes
and returns all memory that is not mapped.

Patch:
==========diff --git a/arch/x86/xen/enlighten.c b/arch/x86/xen/enlighten.c
index b58e963..acc9166 100644
--- a/arch/x86/xen/enlighten.c
+++ b/arch/x86/xen/enlighten.c
@@ -31,6 +31,7 @@
 #include <xen/interface/version.h>
 #include <xen/interface/physdev.h>
 #include <xen/interface/vcpu.h>
+#include <xen/interface/memory.h>
 #include <xen/features.h>
 #include <xen/page.h>
 #include <xen/hvc-console.h>
@@ -122,6 +123,59 @@ static int have_vcpu_info_placement  #endif
 	;
 
+void __init xen_return_unused_memory(void)
+{
+	static struct e820map holes = {
+		.nr_map = 0
+	};
+	struct xen_memory_reservation reservation = {
+		.address_bits = 0,
+		.extent_order = 0,
+		.domid        = DOMID_SELF
+	};
+	unsigned long last_end = 0;
+	int i;
+
+	for(i=0;i<e820.nr_map;i++) {
+		if (e820.map[i].addr > last_end) {
+			holes.map[holes.nr_map].addr = last_end;
+			holes.map[holes.nr_map].size +				e820.map[i].addr - last_end;
+			holes.nr_map++;
+		}		
+		last_end = e820.map[i].addr + e820.map[i].size;
+	}
+
+	if (last_end < PFN_PHYS((u64)xen_start_info->nr_pages)) {
+		holes.map[holes.nr_map].addr=last_end;
+		holes.map[holes.nr_map].size +			PFN_PHYS((u64)xen_start_info->nr_pages) -
last_end;
+		holes.nr_map++;
+	}
+
+	if (holes.nr_map == 0)
+		return;
+
+	for(i=0;i<holes.nr_map;i++) {
+		unsigned long spfn = holes.map[i].addr >> PAGE_SHIFT;
+		unsigned long epfn = ((holes.map[i].addr + holes.map[i].size) >>
PAGE_SHIFT);
+		int ret;
+
+		if (spfn % PAGE_SIZE != 0)
+			spfn++;
+
+		if (spfn >= epfn)
+			continue;
+
+		set_xen_guest_handle(reservation.extent_start,
+			((unsigned long *)xen_start_info->mfn_list) + spfn);
+
+		reservation.nr_extents = epfn - spfn;
+		ret = HYPERVISOR_memory_op(XENMEM_decrease_reservation,
+			&reservation);
+		BUG_ON (ret != epfn - spfn);
+	}
+}
 
 static void xen_vcpu_setup(int cpu)
 {
@@ -1057,6 +1111,8 @@ static __init void xen_post_allocator_init(void)
 	SetPagePinned(virt_to_page(level3_user_vsyscall));
 #endif
 	xen_mark_init_mm_pinned();
+
+/*	xen_return_unused_memory(); */
 }
 
 /* This is called once we have the cpu_possible_map */
-- 
Miroslav Rezanina
Software Engineer - Virtualization Team - XEN kernel


_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xensource.com
http://lists.xensource.com/xen-devel

. snip ..
>  static void xen_vcpu_setup(int cpu)
>  {
> @@ -1057,6 +1111,8 @@ static __init void xen_post_allocator_init(void)
>  	SetPagePinned(virt_to_page(level3_user_vsyscall));
>  #endif
>  	xen_mark_init_mm_pinned();
> +
> +/*	xen_return_unused_memory(); */
Should this be commented out? Or do you plan to provide another
patch to enable this after extensive testing?

_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xensource.com
http://lists.xensource.com/xen-devel

----- "Konrad Rzeszutek Wilk" <konrad.wilk@oracle.com> wrote:
> From: "Konrad Rzeszutek Wilk" <konrad.wilk@oracle.com>
> To: "Miroslav Rezanina" <mrezanin@redhat.com>
> Cc: "Jeremy Fitzhardinge" <jeremy@goop.org>,
xen-devel@lists.xensource.com, linux-kernel@vger.kernel.org
> Sent: Wednesday, September 16, 2009 3:24:56 PM GMT +01:00 Amsterdam /
Berlin / Bern / Rome / Stockholm / Vienna
> Subject: Re: [Xen-devel] Re: [PATCH][v2.6.29][XEN] Return unused memory to
hypervisor
>
> . snip ..
> >  static void xen_vcpu_setup(int cpu)
> >  {
> > @@ -1057,6 +1111,8 @@ static __init void
> xen_post_allocator_init(void)
> >  	SetPagePinned(virt_to_page(level3_user_vsyscall));
> >  #endif
> >  	xen_mark_init_mm_pinned();
> > +
> > +/*	xen_return_unused_memory(); */
> 
> Should this be commented out? Or do you plan to provide another
> patch to enable this after extensive testing?
> 
Ups.....This should not be commented out. I will resent the patch with
correct (uncommented) line.
> _______________________________________________
> Xen-devel mailing list
> Xen-devel@lists.xensource.com
> http://lists.xensource.com/xen-devel
-- 
Miroslav Rezanina
Software Engineer - Virtualization Team - XEN kernel


_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xensource.com
http://lists.xensource.com/xen-devel

On 09/16/09 21:43, Miroslav Rezanina wrote:
> Ups.....This should not be commented out. I will resent the patch with
> correct (uncommented) line.
I''ve already applied a variant of your patch.  Check the
rebase/core-freemem branch in xen.git.

    J


_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xensource.com
http://lists.xensource.com/xen-devel

Hi Jeremy,
I check your application. It won''t work correctly. You are calling
return_unused_memory in xen_memory_setup that is too soon.
When you are calling it, there is no hole in map in any case. Calling should be
in xen_post_allocator_init function (in arch/x86/xen/mmu.c).

Mirek
----- "Jeremy Fitzhardinge" <jeremy@goop.org> wrote:
> From: "Jeremy Fitzhardinge" <jeremy@goop.org>
> To: "Miroslav Rezanina" <mrezanin@redhat.com>
> Cc: "Konrad Rzeszutek Wilk" <konrad.wilk@oracle.com>,
xen-devel@lists.xensource.com, linux-kernel@vger.kernel.org
> Sent: Thursday, September 17, 2009 6:54:56 AM GMT +01:00 Amsterdam / Berlin
/ Bern / Rome / Stockholm / Vienna
> Subject: Re: [Xen-devel] Re: [PATCH][v2.6.29][XEN] Return unused memory to
hypervisor
>
> On 09/16/09 21:43, Miroslav Rezanina wrote:
> > Ups.....This should not be commented out. I will resent the patch
> with
> > correct (uncommented) line.
> 
> I''ve already applied a variant of your patch.  Check the
> rebase/core-freemem branch in xen.git.
> 
>     J
> 
> --
> To unsubscribe from this list: send the line "unsubscribe
> linux-kernel" in
> the body of a message to majordomo@vger.kernel.org
> More majordomo info at  http://vger.kernel.org/majordomo-info.html
> Please read the FAQ at  http://www.tux.org/lkml/
-- 
Miroslav Rezanina
Software Engineer - Virtualization Team - XEN kernel


_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xensource.com
http://lists.xensource.com/xen-devel

On 09/17/09 01:14, Miroslav Rezanina wrote:
> Hi Jeremy,
> I check your application. It won''t work correctly. You are calling
return_unused_memory in xen_memory_setup that is too soon.
> When you are calling it, there is no hole in map in any case. Calling
should be in xen_post_allocator_init function (in arch/x86/xen/mmu.c).
>   
OK.  Where does the e820 map get changed in the meantime?

    J

_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xensource.com
http://lists.xensource.com/xen-devel

----- "Jeremy Fitzhardinge" <jeremy@goop.org> wrote:
> From: "Jeremy Fitzhardinge" <jeremy@goop.org>
> To: "Miroslav Rezanina" <mrezanin@redhat.com>
> Cc: xen-devel@lists.xensource.com, linux-kernel@vger.kernel.org,
"Konrad Rzeszutek Wilk" <konrad.wilk@oracle.com>
> Sent: Thursday, September 17, 2009 6:47:10 PM GMT +01:00 Amsterdam / Berlin
/ Bern / Rome / Stockholm / Vienna
> Subject: Re: [Xen-devel] Re: [PATCH][v2.6.29][XEN] Return unused memory to
hypervisor
>
> On 09/17/09 01:14, Miroslav Rezanina wrote:
> > Hi Jeremy,
> > I check your application. It won''t work correctly. You are
calling
> return_unused_memory in xen_memory_setup that is too soon.
> > When you are calling it, there is no hole in map in any case.
> Calling should be in xen_post_allocator_init function (in
> arch/x86/xen/mmu.c).
> >   
> 
> OK.  Where does the e820 map get changed in the meantime?
> 
xen_memory_setup just set RAM map from 0 to initial nr_pages. Only one place
I''m sure map is changed is
parsing mem parameter. You mentioned that there can be other cases when some
parts of map are removed.
>     J
> 
> _______________________________________________
> Xen-devel mailing list
> Xen-devel@lists.xensource.com
> http://lists.xensource.com/xen-devel
-- 
Miroslav Rezanina
Software Engineer - Virtualization Team - XEN kernel


_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xensource.com
http://lists.xensource.com/xen-devel