Xen devel - Feb 2012 - [PATCH 0 of 2] Virq for low memory condition, V3

This is now a patch series:
- First patch is hypervisor code to implement a virq that will fire upon low 
memory conditions. Integrated feedback from Jan Beulich since version posted on
Feb 23rd.
- Second patch is demo code to leverage the virq form dom0 user-space.

Signed-off-by: Andres Lagar-Cavilla <andres@lagarcavilla.org>
 

 xen/common/page_alloc.c  |   92 +++++++++++++++++++++++++++++
 xen/include/public/xen.h |    1 +
 tools/misc/Makefile      |    7 +-
 tools/misc/lowmemd.c     |  148 +++++++++++++++++++++++++++++++++++++++++++++++
 4 files changed, 246 insertions(+), 2 deletions(-)

xen/common/page_alloc.c  |  92 ++++++++++++++++++++++++++++++++++++++++++++++++
 xen/include/public/xen.h |   1 +
 2 files changed, 93 insertions(+), 0 deletions(-)


When a low memory threshold on the Xen heap is reached, we fire a global dom0
virq. If someone''s listening, they can free up some more memory. The
low
threshold is configurable via the command line token
''low_mem_virq_limit", and
defaults to 64MiB.

We define a new virq VIRQ_ENOMEM. Potential listeners include squeezed,
xenballoond, or anything else that can be fired through xencommons.

We error-check the low mem virq against initial available heap (after dom0
allocation), to avoid firing immediately.

Virq issuing is controlled by a hysteresis algorithm: when memory dips below a
threshold, the virq is issued and the next virq will fire when memory shrinks
another order of magnitude. The virq will not fire again in the current
"band"
until memory grows over the next higher order of magnitude.

Signed-off-by: Andres Lagar-Cavilla <andres@lagarcavilla.org>

diff -r 0696aa1de7c2 -r c44e9fe0ecca xen/common/page_alloc.c
--- a/xen/common/page_alloc.c
+++ b/xen/common/page_alloc.c
@@ -35,6 +35,7 @@
 #include <xen/perfc.h>
 #include <xen/numa.h>
 #include <xen/nodemask.h>
+#include <xen/event.h>
 #include <xen/tmem.h>
 #include <xen/tmem_xen.h>
 #include <public/sysctl.h>
@@ -300,6 +301,91 @@ static unsigned long init_node_heap(int 
     return needed;
 }
 
+/* Default to 64 MiB */
+#define DEFAULT_LOW_MEM_VIRQ    (((paddr_t) 64)   << 20)
+#define MAX_LOW_MEM_VIRQ        (((paddr_t) 1024) << 20)
+
+static paddr_t __read_mostly opt_low_mem_virq = DEFAULT_LOW_MEM_VIRQ;
+size_param("low_mem_virq_limit", opt_low_mem_virq);
+
+/* Thresholds to control hysteresis. In pages */
+/* When memory grows above this threshold, reset hysteresis.
+ * -1 initially to not reset until at least one virq issued. */
+static unsigned long low_mem_virq_high      = -1UL;
+/* Threshold at which we issue virq */
+static unsigned long low_mem_virq_th        = 0;
+/* Original threshold after all checks completed */
+static unsigned long low_mem_virq_orig      = 0;
+/* Order for current threshold */
+static unsigned int  low_mem_virq_th_order  = 0;
+
+/* Perform bootstrapping checks and set bounds */
+static void __init setup_low_mem_virq(void)
+{
+    unsigned int order;
+    paddr_t threshold;
+
+    /* Dom0 has already been allocated by now. So check we won''t 
+     * be complaining immediately with whatever''s left of the heap. */
+    threshold = min(opt_low_mem_virq, 
+                    ((paddr_t) total_avail_pages) << PAGE_SHIFT);
+
+    /* Then, cap to some predefined maximum */
+    threshold = min(threshold, MAX_LOW_MEM_VIRQ);
+
+    /* If the user specified no knob, and we are at the current available 
+     * level, halve the threshold. */
+    if ( (opt_low_mem_virq == DEFAULT_LOW_MEM_VIRQ)
+            && (threshold == (((paddr_t) total_avail_pages) <<
PAGE_SHIFT)) )
+        threshold >>= 1;
+
+    /* Threshold bytes -> pages */
+    low_mem_virq_th = threshold >> PAGE_SHIFT;
+
+    /* Next, round the threshold down to the next order */
+    order = get_order_from_pages(low_mem_virq_th);
+    if ( (1UL << order) > low_mem_virq_th ) 
+        order--;
+
+    /* Set bounds, ready to go */
+    low_mem_virq_th = low_mem_virq_orig = 1UL << order;
+    low_mem_virq_th_order = order;
+
+    printk("Initial low memory virq threshold set at 0x%lx pages.\n",
+            low_mem_virq_th);
+}
+
+static void check_low_mem_virq(void)
+{
+    if ( total_avail_pages <= low_mem_virq_th )
+    {
+        send_global_virq(VIRQ_ENOMEM);
+
+        /* Update thresholds. Next warning will be when we drop below
+         * next order. However, we wait until we grow beyond one
+         * order above us to complain again at the current order */
+        low_mem_virq_high   = 1UL << (low_mem_virq_th_order + 1);
+        if ( low_mem_virq_th_order > 0 )
+            low_mem_virq_th_order--;
+        low_mem_virq_th     = 1UL << low_mem_virq_th_order;
+        return;
+    }
+
+    if ( total_avail_pages >= low_mem_virq_high )
+    {
+        /* Reset hysteresis. Bring threshold up one order.
+         * If we are back where originally set, set high
+         * threshold to -1 to avoid further growth of 
+         * virq threshold. */
+        low_mem_virq_th_order++;
+        low_mem_virq_th = 1UL << low_mem_virq_th_order;
+        if ( low_mem_virq_th == low_mem_virq_orig )
+            low_mem_virq_high = -1UL;
+        else
+            low_mem_virq_high = 1UL << (low_mem_virq_th_order + 2);
+    }
+}
+
 /* Allocate 2^@order contiguous pages. */
 static struct page_info *alloc_heap_pages(
     unsigned int zone_lo, unsigned int zone_hi,
@@ -420,6 +506,8 @@ static struct page_info *alloc_heap_page
     total_avail_pages -= request;
     ASSERT(total_avail_pages >= 0);
 
+    check_low_mem_virq();
+
     if ( d != NULL )
         d->last_alloc_node = node;
 
@@ -1022,6 +1110,10 @@ void __init scrub_heap_pages(void)
     }
 
     printk("done.\n");
+
+    /* Now that the heap is initialized, run checks and set bounds
+     * for the low mem virq algorithm. */
+    setup_low_mem_virq();
 }
 
 
diff -r 0696aa1de7c2 -r c44e9fe0ecca xen/include/public/xen.h
--- a/xen/include/public/xen.h
+++ b/xen/include/public/xen.h
@@ -157,6 +157,7 @@ DEFINE_XEN_GUEST_HANDLE(xen_pfn_t);
 #define VIRQ_PCPU_STATE 9  /* G. (DOM0) PCPU state changed                   */
 #define VIRQ_MEM_EVENT  10 /* G. (DOM0) A memory event has occured           */
 #define VIRQ_XC_RESERVED 11 /* G. Reserved for XenClient                     */
+#define VIRQ_ENOMEM     12 /* G. (DOM0) Low on heap memory       */
 
 /* Architecture-specific VIRQ definitions. */
 #define VIRQ_ARCH_0    16

tools/misc/Makefile  |    7 +-
 tools/misc/lowmemd.c |  148 +++++++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 153 insertions(+), 2 deletions(-)


Signed-off-by: Andres Lagar-Cavilla <andres@lagarcavilla.org>

diff -r c44e9fe0ecca -r 4300b0d76575 tools/misc/Makefile
--- a/tools/misc/Makefile
+++ b/tools/misc/Makefile
@@ -9,7 +9,7 @@ CFLAGS += $(CFLAGS_xeninclude)
 HDRS     = $(wildcard *.h)
 
 TARGETS-y := xenperf xenpm xen-tmem-list-parse gtraceview gtracestat
xenlockprof xenwatchdogd
-TARGETS-$(CONFIG_X86) += xen-detect xen-hvmctx xen-hvmcrash
+TARGETS-$(CONFIG_X86) += xen-detect xen-hvmctx xen-hvmcrash lowmemd
 TARGETS-$(CONFIG_MIGRATE) += xen-hptool
 TARGETS := $(TARGETS-y)
 
@@ -21,7 +21,7 @@ INSTALL_BIN-y := xencons
 INSTALL_BIN-$(CONFIG_X86) += xen-detect
 INSTALL_BIN := $(INSTALL_BIN-y)
 
-INSTALL_SBIN-y := xm xen-bugtool xen-python-path xend xenperf xsview xenpm
xen-tmem-list-parse gtraceview gtracestat xenlockprof xenwatchdogd xen-ringwatch
+INSTALL_SBIN-y := xm xen-bugtool xen-python-path xend xenperf xsview xenpm
xen-tmem-list-parse gtraceview gtracestat xenlockprof xenwatchdogd xen-ringwatch
lowmemd
 INSTALL_SBIN-$(CONFIG_X86) += xen-hvmctx xen-hvmcrash
 INSTALL_SBIN-$(CONFIG_MIGRATE) += xen-hptool
 INSTALL_SBIN := $(INSTALL_SBIN-y)
@@ -70,6 +70,9 @@ xen-hptool: xen-hptool.o
 xenwatchdogd: xenwatchdogd.o
 	$(CC) $(LDFLAGS) -o $@ $< $(LDLIBS_libxenctrl) $(APPEND_LDFLAGS)
 
+lowmemd: lowmemd.o
+	$(CC) $(LDFLAGS) -o $@ $< $(LDLIBS_libxenctrl) $(LDLIBS_libxenstore)
$(APPEND_LDFLAGS)
+
 gtraceview: gtraceview.o
 	$(CC) $(LDFLAGS) -o $@ $< $(CURSES_LIBS) $(APPEND_LDFLAGS)
 
diff -r c44e9fe0ecca -r 4300b0d76575 tools/misc/lowmemd.c
--- /dev/null
+++ b/tools/misc/lowmemd.c
@@ -0,0 +1,148 @@
+/*
+ * lowmemd: demo VIRQ_ENOMEM
+ * Andres Lagar-Cavilla (GridCentric Inc.)
+ */
+
+#include <stdio.h>
+#include <xenctrl.h>
+#include <xs.h>
+#include <stdlib.h>
+#include <string.h>
+
+static evtchn_port_t virq_port      = -1;
+static xc_evtchn *xce_handle        = NULL;
+static xc_interface *xch            = NULL;
+static struct xs_handle *xs_handle  = NULL;
+
+void cleanup(void)
+{
+    if (virq_port > -1)
+        xc_evtchn_unbind(xce_handle, virq_port);
+    if (xce_handle)
+        xc_evtchn_close(xce_handle);
+    if (xch)
+        xc_interface_close(xch);
+    if (xs_handle)
+        xs_daemon_close(xs_handle);
+}
+
+/* Never shrink dom0 below 1 GiB */
+#define DOM0_FLOOR  (1 << 30)
+#define DOM0_FLOOR_PG   ((DOM0_FLOOR) >> 12)
+
+/* Act if free memory is less than 92 MiB */
+#define THRESHOLD   (92 << 20)
+#define THRESHOLD_PG    ((THRESHOLD) >> 12)
+
+#define BUFSZ 512
+void handle_low_mem(void)
+{
+    xc_dominfo_t  dom0_info;
+    xc_physinfo_t info;
+    unsigned long long free_pages, dom0_pages, diff, dom0_target;
+    char data[BUFSZ], error[BUFSZ];
+
+    if (xc_physinfo(xch, &info) < 0)
+    {
+        perror("Getting physinfo failed");
+        return;
+    }
+
+    free_pages = (unsigned long long) info.free_pages;
+    printf("Available free pages: 0x%llx:%llux\n",
+            free_pages, free_pages);
+
+    /* Don''t do anything if we have more than the threshold free */
+    if ( free_pages >= THRESHOLD_PG )
+        return;
+    diff = THRESHOLD_PG - free_pages; 
+
+    if (xc_domain_getinfo(xch, 0, 1, &dom0_info) < 1)
+    {
+        perror("Failed to get dom0 info");
+        return;
+    }
+
+    dom0_pages = (unsigned long long) dom0_info.nr_pages;
+    printf("Dom0 pages: 0x%llx:%llu\n", dom0_pages, dom0_pages);
+    dom0_target = dom0_pages - diff;
+    if (dom0_target <= DOM0_FLOOR_PG)
+        return;
+
+    printf("Shooting for dom0 target 0x%llx:%llu\n", 
+            dom0_target, dom0_target);
+
+    snprintf(data, BUFSZ, "%llu", dom0_target);
+    if (!xs_write(xs_handle, XBT_NULL, 
+            "/local/domain/0/memory/target", data, strlen(data)))
+    {
+        snprintf(error, BUFSZ,"Failed to write target %s to
xenstore", data);
+        perror(error);
+    }
+}
+
+int main(int argc, char *argv[])
+{
+    int rc;
+
+    atexit(cleanup);
+
+	xch = xc_interface_open(NULL, NULL, 0);
+	if (xch == NULL)
+    {
+        perror("Failed to open xc interface");
+        return 1;
+    }
+
+	xce_handle = xc_evtchn_open(NULL, 0);
+	if (xce_handle == NULL)
+    {
+        perror("Failed to open evtchn device");
+        return 2;
+    }
+
+    xs_handle = xs_daemon_open();
+    if (xs_handle == NULL)
+    {
+        perror("Failed to open xenstore connection");
+        return 3;
+    }
+
+	if ((rc = xc_evtchn_bind_virq(xce_handle, VIRQ_ENOMEM)) == -1)
+    {
+        perror("Failed to bind to domain exception virq port");
+        return 4;
+    }
+
+    virq_port = rc;
+    
+    while(1)
+    {
+        evtchn_port_t port;
+
+        if ((port = xc_evtchn_pending(xce_handle)) == -1)
+        {
+            perror("Failed to listen for pending event channel");
+            return 5;
+        }
+
+        if (port != virq_port)
+        {
+            char data[BUFSZ];
+            snprintf(data, BUFSZ, "Wrong port, got %d expected %d",
port, virq_port);
+            perror(data);
+            return 6;
+        }
+
+        if (xc_evtchn_unmask(xce_handle, port) == -1)
+        {
+            perror("Failed to unmask port");
+            return 7;
+        }
+
+        printf("Got a virq kick, time to get work\n");
+        handle_low_mem();
+    }
+
+    return 0;
+}

> From: Andres Lagar-Cavilla [mailto:andres@lagarcavilla.org]
> Sent: Tuesday, February 28, 2012 2:56 PM
> To: xen-devel@lists.xensource.com
> Cc: ian.campbell@citrix.com; andres@gridcentric.ca; tim@xen.org;
JBeulich@suse.com;
> ian.jackson@citrix.com; adin@gridcentric.ca
> Subject: [Xen-devel] [PATCH 1 of 2] Global virq for low memory situations
> 
>  xen/common/page_alloc.c  |  92
++++++++++++++++++++++++++++++++++++++++++++++++
>  xen/include/public/xen.h |   1 +
>  2 files changed, 93 insertions(+), 0 deletions(-)
> 
> 
> When a low memory threshold on the Xen heap is reached, we fire a global
dom0
> virq. If someone''s listening, they can free up some more memory.
The low
> threshold is configurable via the command line token
''low_mem_virq_limit", and
> defaults to 64MiB.
> 
> We define a new virq VIRQ_ENOMEM. Potential listeners include squeezed,
> xenballoond, or anything else that can be fired through xencommons.
> 
> We error-check the low mem virq against initial available heap (after dom0
> allocation), to avoid firing immediately.
> 
> Virq issuing is controlled by a hysteresis algorithm: when memory dips
below a
> threshold, the virq is issued and the next virq will fire when memory
shrinks
> another order of magnitude. The virq will not fire again in the current
"band"
> until memory grows over the next higher order of magnitude.
> 
> Signed-off-by: Andres Lagar-Cavilla <andres@lagarcavilla.org>
Sorry to be late to the party here, but does this take into
account any fragmentation, i.e. does it reflect that allocations
of order==0 will succeed but order>0 might fail?  Should it?

Or maybe Jan''s recent work has eliminated all but the corner
cases of order>0 allocations?  (/me crosses fingers)

Thanks,
Dan

>> From: Andres Lagar-Cavilla [mailto:andres@lagarcavilla.org]
>> Sent: Tuesday, February 28, 2012 2:56 PM
>> To: xen-devel@lists.xensource.com
>> Cc: ian.campbell@citrix.com; andres@gridcentric.ca; tim@xen.org;
>> JBeulich@suse.com;
>> ian.jackson@citrix.com; adin@gridcentric.ca
>> Subject: [Xen-devel] [PATCH 1 of 2] Global virq for low memory
>> situations
>>
>>  xen/common/page_alloc.c  |  92
>> ++++++++++++++++++++++++++++++++++++++++++++++++
>>  xen/include/public/xen.h |   1 +
>>  2 files changed, 93 insertions(+), 0 deletions(-)
>>
>>
>> When a low memory threshold on the Xen heap is reached, we fire a
global
>> dom0
>> virq. If someone''s listening, they can free up some more
memory. The low
>> threshold is configurable via the command line token
>> ''low_mem_virq_limit", and
>> defaults to 64MiB.
>>
>> We define a new virq VIRQ_ENOMEM. Potential listeners include squeezed,
>> xenballoond, or anything else that can be fired through xencommons.
>>
>> We error-check the low mem virq against initial available heap (after
>> dom0
>> allocation), to avoid firing immediately.
>>
>> Virq issuing is controlled by a hysteresis algorithm: when memory dips
>> below a
>> threshold, the virq is issued and the next virq will fire when memory
>> shrinks
>> another order of magnitude. The virq will not fire again in the current
>> "band"
>> until memory grows over the next higher order of magnitude.
>>
>> Signed-off-by: Andres Lagar-Cavilla <andres@lagarcavilla.org>
>
> Sorry to be late to the party here, but does this take into
> account any fragmentation, i.e. does it reflect that allocations
> of order==0 will succeed but order>0 might fail?  Should it?
Not really concerned about fragmentation. Simply looking at overall count.
Andres
>
> Or maybe Jan''s recent work has eliminated all but the corner
> cases of order>0 allocations?  (/me crosses fingers)
>
> Thanks,
> Dan
>

>>> On 28.02.12 at 22:56, Andres Lagar-Cavilla
<andres@lagarcavilla.org> wrote:
> @@ -300,6 +301,91 @@ static unsigned long init_node_heap(int 
>      return needed;
>  }
>  
> +/* Default to 64 MiB */
> +#define DEFAULT_LOW_MEM_VIRQ    (((paddr_t) 64)   << 20)
> +#define MAX_LOW_MEM_VIRQ        (((paddr_t) 1024) << 20)
> +
> +static paddr_t __read_mostly opt_low_mem_virq = DEFAULT_LOW_MEM_VIRQ;
> +size_param("low_mem_virq_limit", opt_low_mem_virq);
> +
> +/* Thresholds to control hysteresis. In pages */
> +/* When memory grows above this threshold, reset hysteresis.
> + * -1 initially to not reset until at least one virq issued. */
> +static unsigned long low_mem_virq_high      = -1UL;
> +/* Threshold at which we issue virq */
> +static unsigned long low_mem_virq_th        = 0;
> +/* Original threshold after all checks completed */
> +static unsigned long low_mem_virq_orig      = 0;
> +/* Order for current threshold */
> +static unsigned int  low_mem_virq_th_order  = 0;
> +
> +/* Perform bootstrapping checks and set bounds */
> +static void __init setup_low_mem_virq(void)
> +{
> +    unsigned int order;
> +    paddr_t threshold;
> +
> +    /* Dom0 has already been allocated by now. So check we won''t 
> +     * be complaining immediately with whatever''s left of the
heap. */
> +    threshold = min(opt_low_mem_virq, 
> +                    ((paddr_t) total_avail_pages) << PAGE_SHIFT);
> +
> +    /* Then, cap to some predefined maximum */
> +    threshold = min(threshold, MAX_LOW_MEM_VIRQ);
> +
> +    /* If the user specified no knob, and we are at the current available 
> +     * level, halve the threshold. */
> +    if ( (opt_low_mem_virq == DEFAULT_LOW_MEM_VIRQ)
This condition being true is not an indication that there was nothing
specified on the command line.
> +            && (threshold == (((paddr_t) total_avail_pages)
<< PAGE_SHIFT)) )
> +        threshold >>= 1;
> +
> +    /* Threshold bytes -> pages */
> +    low_mem_virq_th = threshold >> PAGE_SHIFT;
> +
> +    /* Next, round the threshold down to the next order */
> +    order = get_order_from_pages(low_mem_virq_th);
This has undefined (and unexpected by you) behavior if
low_mem_virq_th is zero (which I would think ought to mean "disable
this whole logic").
> +    if ( (1UL << order) > low_mem_virq_th ) 
> +        order--;
> +
> +    /* Set bounds, ready to go */
> +    low_mem_virq_th = low_mem_virq_orig = 1UL << order;
> +    low_mem_virq_th_order = order;
> +
> +    printk("Initial low memory virq threshold set at 0x%lx
pages.\n",
> +            low_mem_virq_th);
> +}
> +
> +static void check_low_mem_virq(void)
> +{
> +    if ( total_avail_pages <= low_mem_virq_th )
> +    {
> +        send_global_virq(VIRQ_ENOMEM);
> +
> +        /* Update thresholds. Next warning will be when we drop below
> +         * next order. However, we wait until we grow beyond one
> +         * order above us to complain again at the current order */
> +        low_mem_virq_high   = 1UL << (low_mem_virq_th_order + 1);
> +        if ( low_mem_virq_th_order > 0 )
> +            low_mem_virq_th_order--;
> +        low_mem_virq_th     = 1UL << low_mem_virq_th_order;
> +        return;
> +    }
> +
> +    if ( total_avail_pages >= low_mem_virq_high )
> +    {
> +        /* Reset hysteresis. Bring threshold up one order.
> +         * If we are back where originally set, set high
> +         * threshold to -1 to avoid further growth of 
> +         * virq threshold. */
> +        low_mem_virq_th_order++;
> +        low_mem_virq_th = 1UL << low_mem_virq_th_order;
> +        if ( low_mem_virq_th == low_mem_virq_orig )
> +            low_mem_virq_high = -1UL;
> +        else
> +            low_mem_virq_high = 1UL << (low_mem_virq_th_order + 2);
> +    }
> +}
> +
>  /* Allocate 2^@order contiguous pages. */
>  static struct page_info *alloc_heap_pages(
>      unsigned int zone_lo, unsigned int zone_hi,
Also, could you please check your patches for not introducing trailing
whitespace?

Jan

>>> On 29.02.12 at 00:50, Dan Magenheimer
<dan.magenheimer@oracle.com> wrote:
> Or maybe Jan''s recent work has eliminated all but the corner
> cases of order>0 allocations?  (/me crosses fingers)
I''m unaware of remaining violators of this, but I''m also not
constantly
monitoring.

Jan

>>> On 28.02.12 at 22:56, Andres Lagar-Cavilla
<andres@lagarcavilla.org> wrote:
Thanks for contributing this.
> --- a/tools/misc/Makefile
> +++ b/tools/misc/Makefile
> @@ -9,7 +9,7 @@ CFLAGS += $(CFLAGS_xeninclude)
>  HDRS     = $(wildcard *.h)
>  
>  TARGETS-y := xenperf xenpm xen-tmem-list-parse gtraceview gtracestat 
> xenlockprof xenwatchdogd
> -TARGETS-$(CONFIG_X86) += xen-detect xen-hvmctx xen-hvmcrash
> +TARGETS-$(CONFIG_X86) += xen-detect xen-hvmctx xen-hvmcrash lowmemd
If this should get committed (which I''m in favor of at least as a first
step, to get more advanced over time if needed), we would certainly
want to prefix the binary with xen- (since that''s what it''s
for, it''s not
dealing with domain side low memory conditions).

Jan
>  TARGETS-$(CONFIG_MIGRATE) += xen-hptool
>  TARGETS := $(TARGETS-y)
>  
> @@ -21,7 +21,7 @@ INSTALL_BIN-y := xencons
>  INSTALL_BIN-$(CONFIG_X86) += xen-detect
>  INSTALL_BIN := $(INSTALL_BIN-y)
>  
> -INSTALL_SBIN-y := xm xen-bugtool xen-python-path xend xenperf xsview xenpm
> xen-tmem-list-parse gtraceview gtracestat xenlockprof xenwatchdogd 
> xen-ringwatch
> +INSTALL_SBIN-y := xm xen-bugtool xen-python-path xend xenperf xsview xenpm
> xen-tmem-list-parse gtraceview gtracestat xenlockprof xenwatchdogd 
> xen-ringwatch lowmemd
>  INSTALL_SBIN-$(CONFIG_X86) += xen-hvmctx xen-hvmcrash
>  INSTALL_SBIN-$(CONFIG_MIGRATE) += xen-hptool
>  INSTALL_SBIN := $(INSTALL_SBIN-y)
> @@ -70,6 +70,9 @@ xen-hptool: xen-hptool.o
>  xenwatchdogd: xenwatchdogd.o
>  	$(CC) $(LDFLAGS) -o $@ $< $(LDLIBS_libxenctrl) $(APPEND_LDFLAGS)
>  
> +lowmemd: lowmemd.o
> +	$(CC) $(LDFLAGS) -o $@ $< $(LDLIBS_libxenctrl) $(LDLIBS_libxenstore) 
> $(APPEND_LDFLAGS)
> +
>  gtraceview: gtraceview.o
>  	$(CC) $(LDFLAGS) -o $@ $< $(CURSES_LIBS) $(APPEND_LDFLAGS)
>  
> diff -r c44e9fe0ecca -r 4300b0d76575 tools/misc/lowmemd.c
> --- /dev/null
> +++ b/tools/misc/lowmemd.c
> @@ -0,0 +1,148 @@
> +/*
> + * lowmemd: demo VIRQ_ENOMEM
> + * Andres Lagar-Cavilla (GridCentric Inc.)
> + */
> +
> +#include <stdio.h>
> +#include <xenctrl.h>
> +#include <xs.h>
> +#include <stdlib.h>
> +#include <string.h>
> +
> +static evtchn_port_t virq_port      = -1;
> +static xc_evtchn *xce_handle        = NULL;
> +static xc_interface *xch            = NULL;
> +static struct xs_handle *xs_handle  = NULL;
> +
> +void cleanup(void)
> +{
> +    if (virq_port > -1)
> +        xc_evtchn_unbind(xce_handle, virq_port);
> +    if (xce_handle)
> +        xc_evtchn_close(xce_handle);
> +    if (xch)
> +        xc_interface_close(xch);
> +    if (xs_handle)
> +        xs_daemon_close(xs_handle);
> +}
> +
> +/* Never shrink dom0 below 1 GiB */
> +#define DOM0_FLOOR  (1 << 30)
> +#define DOM0_FLOOR_PG   ((DOM0_FLOOR) >> 12)
> +
> +/* Act if free memory is less than 92 MiB */
> +#define THRESHOLD   (92 << 20)
> +#define THRESHOLD_PG    ((THRESHOLD) >> 12)
> +
> +#define BUFSZ 512
> +void handle_low_mem(void)
> +{
> +    xc_dominfo_t  dom0_info;
> +    xc_physinfo_t info;
> +    unsigned long long free_pages, dom0_pages, diff, dom0_target;
> +    char data[BUFSZ], error[BUFSZ];
> +
> +    if (xc_physinfo(xch, &info) < 0)
> +    {
> +        perror("Getting physinfo failed");
> +        return;
> +    }
> +
> +    free_pages = (unsigned long long) info.free_pages;
> +    printf("Available free pages: 0x%llx:%llux\n",
> +            free_pages, free_pages);
> +
> +    /* Don''t do anything if we have more than the threshold free
*/
> +    if ( free_pages >= THRESHOLD_PG )
> +        return;
> +    diff = THRESHOLD_PG - free_pages; 
> +
> +    if (xc_domain_getinfo(xch, 0, 1, &dom0_info) < 1)
> +    {
> +        perror("Failed to get dom0 info");
> +        return;
> +    }
> +
> +    dom0_pages = (unsigned long long) dom0_info.nr_pages;
> +    printf("Dom0 pages: 0x%llx:%llu\n", dom0_pages, dom0_pages);
> +    dom0_target = dom0_pages - diff;
> +    if (dom0_target <= DOM0_FLOOR_PG)
> +        return;
> +
> +    printf("Shooting for dom0 target 0x%llx:%llu\n", 
> +            dom0_target, dom0_target);
> +
> +    snprintf(data, BUFSZ, "%llu", dom0_target);
> +    if (!xs_write(xs_handle, XBT_NULL, 
> +            "/local/domain/0/memory/target", data,
strlen(data)))
> +    {
> +        snprintf(error, BUFSZ,"Failed to write target %s to
xenstore",
> data);
> +        perror(error);
> +    }
> +}
> +
> +int main(int argc, char *argv[])
> +{
> +    int rc;
> +
> +    atexit(cleanup);
> +
> +	xch = xc_interface_open(NULL, NULL, 0);
> +	if (xch == NULL)
> +    {
> +        perror("Failed to open xc interface");
> +        return 1;
> +    }
> +
> +	xce_handle = xc_evtchn_open(NULL, 0);
> +	if (xce_handle == NULL)
> +    {
> +        perror("Failed to open evtchn device");
> +        return 2;
> +    }
> +
> +    xs_handle = xs_daemon_open();
> +    if (xs_handle == NULL)
> +    {
> +        perror("Failed to open xenstore connection");
> +        return 3;
> +    }
> +
> +	if ((rc = xc_evtchn_bind_virq(xce_handle, VIRQ_ENOMEM)) == -1)
> +    {
> +        perror("Failed to bind to domain exception virq port");
> +        return 4;
> +    }
> +
> +    virq_port = rc;
> +    
> +    while(1)
> +    {
> +        evtchn_port_t port;
> +
> +        if ((port = xc_evtchn_pending(xce_handle)) == -1)
> +        {
> +            perror("Failed to listen for pending event
channel");
> +            return 5;
> +        }
> +
> +        if (port != virq_port)
> +        {
> +            char data[BUFSZ];
> +            snprintf(data, BUFSZ, "Wrong port, got %d expected
%d", port,
> virq_port);
> +            perror(data);
> +            return 6;
> +        }
> +
> +        if (xc_evtchn_unmask(xce_handle, port) == -1)
> +        {
> +            perror("Failed to unmask port");
> +            return 7;
> +        }
> +
> +        printf("Got a virq kick, time to get work\n");
> +        handle_low_mem();
> +    }
> +
> +    return 0;
> +}