Ryan Harper
2005-Dec-16 23:09 UTC
[Xen-devel] [PATCH] 4/7 xen: Add basic NUMA support - Debug Keyhandler
This patch adds a new keyhandler function, dump_numa_stats, which is
triggered on the Xen debug console by pressing ''n''. This
function will
dump a collection of NUMA-related information. Such information is
useful when debugging domain memory placement on NUMA systems.
In a future patch we would like to export this information via a
hypercall so it can be collected automatically rather than
interactively.
Here is an example of the output on a Dual Opteron (2 Node, 2 CPU, with
one guest running).
(XEN) ''n'' pressed -> dumping numa stats
(XEN) ZONE:XEN NUMA hits 33 misses 4 allocs 41 frees 2579
(XEN) ZONE:DOM NUMA hits 17028 misses 17292 allocs 34321 frees 410828
(XEN) ***********************************
(XEN) stats for dom 0:
(XEN) vcpu[0] -> cpu[0]
(XEN) vcpu[1] -> cpu[1]
(XEN) numa hits : 677
(XEN) numa misses: 912
(XEN) page allocs: 1589
(XEN) total pages currently possessed: 131072
(XEN) total xen heap pages : 5
(XEN) pages in node 0: 18520
(XEN) pages in node 1: 112552
(XEN) xenheap pages in node 0: 5
(XEN) xenheap pages in node 1: 0
(XEN) pages not matched: 0
(XEN) ***********************************
(XEN) ***********************************
(XEN) stats for dom 1:
(XEN) vcpu[0] -> cpu[1]
(XEN) vcpu[1] -> cpu[0]
(XEN) vcpu[2] -> cpu[1]
(XEN) vcpu[3] -> cpu[0]
(XEN) numa hits : 16388
(XEN) numa misses: 16383
(XEN) page allocs: 32771
(XEN) total pages currently possessed: 32768
(XEN) total xen heap pages : 5
(XEN) pages in node 0: 0
(XEN) pages in node 1: 32768
(XEN) xenheap pages in node 0: 5
(XEN) xenheap pages in node 1: 0
(XEN) pages not matched: 0
(XEN) ***********************************
(XEN) free pages in node 0: 499800
(XEN) free pages in node 1: 246712
--
Ryan Harper
Software Engineer; Linux Technology Center
IBM Corp., Austin, Tx
(512) 838-9253 T/L: 678-9253
ryanh@us.ibm.com
diffstat output:
common/keyhandler.c | 99 ++++++++++++++++++++++++++++++++++++++++++++++++++++
common/page_alloc.c | 43 ++++++++++++++++++++++
include/xen/numa.h | 2 +
3 files changed, 143 insertions(+), 1 deletion(-)
Signed-off-by: Ryan Harper <ryanh@us.ibm.com>
Signed-off-by: Ryan Grimm <grimm@us.ibm.com>
---
diff -r 31e1def2471e xen/common/keyhandler.c
--- a/xen/common/keyhandler.c Mon Dec 12 19:46:49 2005
+++ b/xen/common/keyhandler.c Mon Dec 12 14:16:30 2005
@@ -15,6 +15,16 @@
#define KEY_MAX 256
#define STR_MAX 64
+#ifdef CONFIG_NUMA
+#include <xen/numa.h>
+/* NUMA/Alloc stats */
+extern unsigned long page_alloc[]; /* total page allocs */
+extern unsigned long page_free[]; /* total page frees */
+extern unsigned long numa_hit[]; /* allocated in intended node */
+extern unsigned long numa_miss[]; /* allocated in non intended node */
+#define MEMZONE_XEN 0
+#define MEMZONE_DOM 1
+#endif
static struct {
union {
@@ -145,6 +155,91 @@
read_unlock(&domlist_lock);
}
+#ifdef CONFIG_NUMA
+static void dump_numa_stats(unsigned char key)
+{
+ struct domain *d = NULL;
+ struct vcpu *v = NULL;
+ struct pfn_info *pg = NULL;
+ int i = 0;
+ u32 free_pages[nodes_detected];
+
+ printk("''%c'' pressed -> dumping numa
stats\n", key);
+ printk("ZONE:XEN NUMA hits %ld misses %ld allocs %ld frees
%ld\n",
+ numa_hit[MEMZONE_XEN],
+ numa_miss[MEMZONE_XEN],
+ page_alloc[MEMZONE_XEN],
+ page_free[MEMZONE_XEN]);
+ printk("ZONE:DOM NUMA hits %ld misses %ld allocs %ld frees
%ld\n",
+ numa_hit[MEMZONE_DOM],
+ numa_miss[MEMZONE_DOM],
+ page_alloc[MEMZONE_DOM],
+ page_free[MEMZONE_DOM]);
+ /* show per-domain vcpu->cpu mappings and numa stats */
+ read_lock(&domlist_lock);
+
+ for_each_domain (d) {
+ u32 domain_page_to_nodes[nodes_detected];
+ u32 xenheap_page_to_nodes[nodes_detected];
+ u32 lost_pages = 0;
+
+ memset(domain_page_to_nodes, 0, sizeof(domain_page_to_nodes));
+ memset(xenheap_page_to_nodes, 0, sizeof(xenheap_page_to_nodes));
+
+ printk("***********************************\n");
+ printk("stats for dom %d:\n", d->domain_id);
+ /* vcpu->cpu mappings */
+ for_each_vcpu (d, v)
+ /* make sure vcpu is up */
+ if (!test_bit(_VCPUF_down, &v->vcpu_flags))
+ printk("vcpu[%d] -> cpu[%d]\n", v->vcpu_id,
v->processor);
+
+ /* numa stats */
+ printk("numa hits : %ld\n", d->numa_hit);
+ printk("numa misses: %ld\n", d->numa_miss);
+ printk("page allocs: %ld\n", d->page_alloc);
+
+ /* page to node mappings */
+ spin_lock(&d->page_alloc_lock);
+ list_for_each_entry(pg, &d->page_list, list) {
+ int nid = page_to_node(pg);
+
+ if (nid >= 0)
+ domain_page_to_nodes[nid]++;
+ else
+ lost_pages++;
+ }
+
+ list_for_each_entry(pg, &d->xenpage_list, list) {
+ int nid = page_to_node(pg);
+
+ if (nid >= 0)
+ xenheap_page_to_nodes[nid]++;
+ else
+ lost_pages++;
+ }
+ spin_unlock(&d->page_alloc_lock);
+ printk("total pages currently possessed: %d\n",
d->tot_pages);
+ printk("total xen heap pages : %d\n",
d->xenheap_pages);
+ for (i = 0; i < nodes_detected; i++) {
+ printk("pages in node %d: %d\n", i,
+ domain_page_to_nodes[i]);
+ }
+ for (i = 0; i < nodes_detected; i++) {
+ printk("xenheap pages in node %d: %d\n", i,
+ xenheap_page_to_nodes[i]);
+ }
+ printk("pages not matched: %d\n", lost_pages);
+ printk("***********************************\n");
+ }
+ read_unlock(&domlist_lock);
+
+ memset(free_pages, 0, sizeof(free_pages));
+
+ for ( i = 0; i < nodes_detected; i++)
+ printk("free pages in node %d: %d\n", i,
node_free_pages(i));
+}
+#endif
extern void dump_runq(unsigned char key);
extern void print_sched_histo(unsigned char key);
extern void reset_sched_histo(unsigned char key);
@@ -187,6 +282,10 @@
''l'', print_sched_histo, "print sched latency
histogram");
register_keyhandler(
''L'', reset_sched_histo, "reset sched latency
histogram");
+#ifdef CONFIG_NUMA
+ register_keyhandler(
+ ''n'', dump_numa_stats, "dump numa stats");
+#endif
register_keyhandler(
''q'', do_task_queues, "dump task queues + guest
state");
register_keyhandler(
diff -r 31e1def2471e xen/common/page_alloc.c
--- a/xen/common/page_alloc.c Mon Dec 12 19:46:49 2005
+++ b/xen/common/page_alloc.c Mon Dec 12 14:16:30 2005
@@ -308,7 +308,7 @@
u64 cpu_end; /* highest end addr near cpu */
u64 start = 0, end = 0, length = (PAGE_SIZE << list_order);
struct pfn_info *pg = NULL;
- struct node_memory_chunk_s *c;
+ struct node_memory_chunk_s *c=NULL;
nid = cpu_to_node[cpu];
cpu_start = node_min_paddr[nid];
@@ -349,6 +349,47 @@
out:
return pg;
+}
+
+/* page to node mapping */
+int page_to_node(struct pfn_info *pg)
+{
+ u64 pg_paddr = page_to_phys(pg);
+ u64 node_start_paddr = 0ULL, node_end_paddr = 0ULL;
+ u8 nid = 0;
+ int i = 0;
+
+ for (i = 0; i < num_memory_chunks; i++) {
+ node_start_paddr = node_memory_chunk[i].start_paddr;
+ node_end_paddr = node_memory_chunk[i].end_paddr;
+ nid = node_memory_chunk[i].nid;
+
+ if (pg_paddr >= node_start_paddr && pg_paddr <=
node_end_paddr)
+ goto found;
+ }
+ return -1;
+
+found:
+ return nid;
+}
+
+int node_free_pages(int nid)
+{
+ int i = 0, j = 0, free_pages = 0;
+ struct pfn_info *pg = NULL;
+ spin_lock(&heap_lock);
+ for ( i = 0; i < NR_ZONES; i++)
+ for ( j = 0; j < MAX_ORDER+1; j++) {
+ /* if page is on free list */
+ list_for_each_entry( pg, &heap[i][j], list) {
+ if ( (pg->count_info & PGC_count_mask) == 0) {
+ if ( page_to_node(pg) == nid)
+ free_pages+=1<<j;
+ }
+ }
+ }
+ spin_unlock(&heap_lock);
+ return free_pages;
}
#endif
diff -r 31e1def2471e xen/include/xen/numa.h
--- a/xen/include/xen/numa.h Mon Dec 12 19:46:49 2005
+++ b/xen/include/xen/numa.h Mon Dec 12 14:16:30 2005
@@ -42,5 +36,7 @@
extern unsigned long numa_miss[]; /* allocated in non intended node */
int numa_init(void);
+int page_to_node(struct pfn_info *pg);
+int node_free_pages(int nid);
#endif /* _LINUX_NUMA_H */
_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xensource.com
http://lists.xensource.com/xen-devel