To reduce the performance side effects of ballooning, use and return
cold pages. To limit the impact scrubbing of these (and other) pages
has on the cache, also implement a dedicated scrubbing function on x86
which uses non-temporal stores (when available).
As usual, written and tested on 2.6.24 and made apply to the 2.6.18
tree without further testing.
Signed-off-by: Jan Beulich <jbeulich@novell.com>
Index: head-2008-01-28/arch/i386/lib/Makefile
==================================================================---
head-2008-01-28.orig/arch/i386/lib/Makefile 2008-01-24 23:58:37.000000000 +0100
+++ head-2008-01-28/arch/i386/lib/Makefile 2008-02-08 10:46:49.000000000 +0100
@@ -7,3 +7,4 @@ include ${srctree}/arch/x86/lib/Makefile
bitops.o
lib-$(CONFIG_X86_USE_3DNOW) += mmx.o
+lib-$(CONFIG_XEN_SCRUB_PAGES) += scrub.o
Index: head-2008-01-28/arch/i386/lib/scrub.c
==================================================================--- /dev/null
1970-01-01 00:00:00.000000000 +0000
+++ head-2008-01-28/arch/i386/lib/scrub.c 2008-02-08 12:30:51.000000000 +0100
@@ -0,0 +1,21 @@
+#include <asm/cpufeature.h>
+#include <asm/page.h>
+#include <asm/processor.h>
+
+void scrub_pages(void *v, unsigned int count)
+{
+ if (likely(cpu_has_xmm2)) {
+ unsigned long n = count * (PAGE_SIZE / sizeof(long) / 4);
+
+ for (; n--; v += sizeof(long) * 4)
+ asm("movnti %1,(%0)\n\t"
+ "movnti %1,%c2(%0)\n\t"
+ "movnti %1,2*%c2(%0)\n\t"
+ "movnti %1,3*%c2(%0)\n\t"
+ : : "r" (v), "r" (0L), "i" (sizeof(long))
+ : "memory");
+ asm volatile("sfence" : : : "memory");
+ } else
+ for (; count--; v += PAGE_SIZE)
+ clear_page(v);
+}
Index: head-2008-01-28/arch/i386/mm/hypervisor.c
==================================================================---
head-2008-01-28.orig/arch/i386/mm/hypervisor.c 2008-02-08 14:29:22.000000000
+0100
+++ head-2008-01-28/arch/i386/mm/hypervisor.c 2008-02-08 11:30:34.000000000
+0100
@@ -281,7 +281,7 @@ int xen_create_contiguous_region(
set_xen_guest_handle(exchange.in.extent_start, in_frames);
set_xen_guest_handle(exchange.out.extent_start, &out_frame);
- scrub_pages(vstart, 1 << order);
+ scrub_pages((void *)vstart, 1 << order);
balloon_lock(flags);
@@ -374,7 +374,7 @@ void xen_destroy_contiguous_region(unsig
set_xen_guest_handle(exchange.in.extent_start, &in_frame);
set_xen_guest_handle(exchange.out.extent_start, out_frames);
- scrub_pages(vstart, 1 << order);
+ scrub_pages((void *)vstart, 1 << order);
balloon_lock(flags);
Index: head-2008-01-28/arch/x86_64/lib/Makefile
==================================================================---
head-2008-01-28.orig/arch/x86_64/lib/Makefile 2008-01-24 23:58:37.000000000
+0100
+++ head-2008-01-28/arch/x86_64/lib/Makefile 2008-02-08 10:46:49.000000000 +0100
@@ -10,3 +10,4 @@ include ${srctree}/arch/x86/lib/Makefile
usercopy.o getuser.o putuser.o \
thunk.o clear_page.o copy_page.o bitstr.o bitops.o
lib-y += memcpy.o memmove.o memset.o copy_user.o
+lib-$(CONFIG_XEN_SCRUB_PAGES) += scrub.o
Index: head-2008-01-28/arch/x86_64/lib/scrub.c
==================================================================--- /dev/null
1970-01-01 00:00:00.000000000 +0000
+++ head-2008-01-28/arch/x86_64/lib/scrub.c 2008-02-08 12:30:51.000000000 +0100
@@ -0,0 +1,1 @@
+#include "../../i386/lib/scrub.c"
Index: head-2008-01-28/drivers/xen/balloon/balloon.c
==================================================================---
head-2008-01-28.orig/drivers/xen/balloon/balloon.c 2008-02-08 14:29:22.000000000
+0100
+++ head-2008-01-28/drivers/xen/balloon/balloon.c 2008-02-08 14:29:36.000000000
+0100
@@ -105,7 +105,7 @@ static struct timer_list balloon_timer;
/* When ballooning out (allocating memory to return to Xen) we don''t
really
want the kernel to try too hard since that can trigger the oom killer. */
#define GFP_BALLOON \
- (GFP_HIGHUSER | __GFP_NOWARN | __GFP_NORETRY | __GFP_NOMEMALLOC)
+ (GFP_HIGHUSER|__GFP_NOWARN|__GFP_NORETRY|__GFP_NOMEMALLOC|__GFP_COLD)
#define PAGE_TO_LIST(p) (&(p)->lru)
#define LIST_TO_PAGE(l) list_entry((l), struct page, lru)
@@ -175,6 +175,17 @@ static struct page *balloon_next_page(st
return LIST_TO_PAGE(next);
}
+static inline void balloon_free_page(struct page *page)
+{
+#ifndef MODULE
+ if (put_page_testzero(page))
+ free_cold_page(page);
+#else
+ /* free_cold_page() is not being exported. */
+ __free_page(page);
+#endif
+}
+
static void balloon_alarm(unsigned long unused)
{
schedule_work(&balloon_worker);
@@ -292,7 +303,7 @@ static int increase_reservation(unsigned
/* Relinquish the page back to the allocator. */
ClearPageReserved(page);
init_page_count(page);
- __free_page(page);
+ balloon_free_page(page);
}
bs.current_pages += nr_pages;
@@ -623,7 +634,8 @@ static int dealloc_pte_fn(
struct page **alloc_empty_pages_and_pagevec(int nr_pages)
{
- unsigned long vaddr, flags;
+ unsigned long flags;
+ void *v;
struct page *page, **pagevec;
int i, ret;
@@ -632,13 +644,12 @@ struct page **alloc_empty_pages_and_page
return NULL;
for (i = 0; i < nr_pages; i++) {
- page = pagevec[i] = alloc_page(GFP_KERNEL);
+ page = pagevec[i] = alloc_page(GFP_KERNEL|__GFP_COLD);
if (page == NULL)
goto err;
- vaddr = (unsigned long)page_address(page);
-
- scrub_pages(vaddr, 1);
+ v = page_address(page);
+ scrub_pages(v, 1);
balloon_lock(flags);
@@ -656,8 +667,9 @@ struct page **alloc_empty_pages_and_page
ret = 0; /* success */
} else {
#ifdef CONFIG_XEN
- ret = apply_to_page_range(&init_mm, vaddr, PAGE_SIZE,
- dealloc_pte_fn, NULL);
+ ret = apply_to_page_range(&init_mm, (unsigned long)v,
+ PAGE_SIZE, dealloc_pte_fn,
+ NULL);
#else
/* Cannot handle non-auto translate mode. */
ret = 1;
@@ -668,7 +679,7 @@ struct page **alloc_empty_pages_and_page
if (ret != 0) {
balloon_unlock(flags);
- __free_page(page);
+ balloon_free_page(page);
goto err;
}
Index: head-2008-01-28/include/asm-i386/mach-xen/asm/hypervisor.h
==================================================================---
head-2008-01-28.orig/include/asm-i386/mach-xen/asm/hypervisor.h 2008-02-08
14:29:22.000000000 +0100
+++ head-2008-01-28/include/asm-i386/mach-xen/asm/hypervisor.h 2008-02-08
10:42:13.000000000 +0100
@@ -121,7 +121,7 @@ int xen_limit_pages_to_max_mfn(
u64 jiffies_to_st(unsigned long jiffies);
#ifdef CONFIG_XEN_SCRUB_PAGES
-#define scrub_pages(_p,_n) memset((void *)(_p), 0, (_n) << PAGE_SHIFT)
+void scrub_pages(void *, unsigned int);
#else
#define scrub_pages(_p,_n) ((void)0)
#endif
_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xensource.com
http://lists.xensource.com/xen-devel