Recent debugging on vhost net zerocopy shows the need of
tracepoints. So to help in vhost{net} debugging and performance
analyzing, the following series adding basic tracepoints to
vhost. Operations of both vhost and vhost_net were traced in current
implementation.
A top-like satistics displaying script were introduced to help the
troubleshooting:
vhost statistics
vhost_virtio_update_used_idx 1215215 0
vhost_virtio_get_vq_desc 1215215 0
vhost_work_queue_wakeup 986808 0
vhost_virtio_signal 811601 0
vhost_net_tx 611457 0
vhost_net_rx 603758 0
vhost_net_tx(datacopy) 601903 0
vhost_work_queue_wakeup(rx_net) 565081 0
vhost_virtio_signal(rx) 461603 0
vhost_work_queue_wakeup(tx_kick) 421718 0
vhost_virtio_update_avail_event 417346 0
vhost_virtio_signal(tx) 349998 0
vhost_work_queue_coalesce 39384 0
vhost_work_queue_coalesce(rx_net) 38677 0
vhost_net_tx(zerocopy) 9554 0
vhost_work_queue_coalesce(tx_kick) 707 0
vhost_work_queue_wakeup(rx_kick) 9 0
TODO:
- performance test
- finalize which should be traced
Reference:
- V1: https://lkml.org/lkml/2012/4/9/478
Jason Wang (4):
vhost: introduce queue_index for tracing
vhost: basic tracepoints
vhost_net: add basic tracepoints for vhost_net
tools: virtio: add a top-like utility for displaying vhost satistics
drivers/vhost/net.c | 7 +
drivers/vhost/net_trace.h | 53 +++++++
drivers/vhost/trace.h | 175 ++++++++++++++++++++++
drivers/vhost/vhost.c | 14 +-
drivers/vhost/vhost.h | 3 +
tools/virtio/vhost_stat | 375 ++++++++++++++++++++++++++++++++++++++++++++++
6 files changed, 626 insertions(+), 1 deletion(-)
create mode 100644 drivers/vhost/net_trace.h
create mode 100644 drivers/vhost/trace.h
create mode 100755 tools/virtio/vhost_stat
--
1.8.3.2
Jason Wang
2014-Mar-21 09:41 UTC
[PATCH RFC V2 1/4] vhost: introduce queue_index for tracing
Signed-off-by: Jason Wang <jasowang at redhat.com>
---
drivers/vhost/net.c | 1 +
drivers/vhost/vhost.h | 3 +++
2 files changed, 4 insertions(+)
diff --git a/drivers/vhost/net.c b/drivers/vhost/net.c
index a0fa5de..85d666c 100644
--- a/drivers/vhost/net.c
+++ b/drivers/vhost/net.c
@@ -708,6 +708,7 @@ static int vhost_net_open(struct inode *inode, struct file
*f)
n->vqs[i].done_idx = 0;
n->vqs[i].vhost_hlen = 0;
n->vqs[i].sock_hlen = 0;
+ n->vqs[i].vq.queue_index = i;
}
vhost_dev_init(dev, vqs, VHOST_NET_VQ_MAX);
diff --git a/drivers/vhost/vhost.h b/drivers/vhost/vhost.h
index 35eeb2a..6e36416 100644
--- a/drivers/vhost/vhost.h
+++ b/drivers/vhost/vhost.h
@@ -96,6 +96,9 @@ struct vhost_virtqueue {
/* Last used index value we have signalled on */
bool signalled_used_valid;
+ /* Queue index used for tracing */
+ u16 queue_index;
+
/* Log writes to used structure. */
bool log_used;
u64 log_addr;
--
1.8.3.2
To help for the performance optimizations and debugging, this patch tracepoints
for vhost. Two kinds of activities were traced: virtio and vhost work
queuing/wakeup.
Signed-off-by: Jason Wang <jasowang at redhat.com>
---
drivers/vhost/net.c | 1 +
drivers/vhost/trace.h | 175 ++++++++++++++++++++++++++++++++++++++++++++++++++
drivers/vhost/vhost.c | 14 +++-
3 files changed, 189 insertions(+), 1 deletion(-)
create mode 100644 drivers/vhost/trace.h
diff --git a/drivers/vhost/net.c b/drivers/vhost/net.c
index 85d666c..7353204 100644
--- a/drivers/vhost/net.c
+++ b/drivers/vhost/net.c
@@ -1,3 +1,4 @@
+
/* Copyright (C) 2009 Red Hat, Inc.
* Author: Michael S. Tsirkin <mst at redhat.com>
*
diff --git a/drivers/vhost/trace.h b/drivers/vhost/trace.h
new file mode 100644
index 0000000..e380942
--- /dev/null
+++ b/drivers/vhost/trace.h
@@ -0,0 +1,175 @@
+#if !defined(_TRACE_VHOST_H) || defined(TRACE_HEADER_MULTI_READ)
+#define _TRACE_VHOST_H
+
+#include <linux/tracepoint.h>
+#include "vhost.h"
+
+#undef TRACE_SYSTEM
+#define TRACE_SYSTEM vhost
+
+/*
+ * Tracepoint for updating used flag.
+ */
+TRACE_EVENT(vhost_virtio_update_used_flags,
+ TP_PROTO(struct vhost_virtqueue *vq),
+ TP_ARGS(vq),
+
+ TP_STRUCT__entry(
+ __field(struct vhost_virtqueue *, vq)
+ __field(u16, queue_index)
+ __field(u16, used_flags)
+ ),
+
+ TP_fast_assign(
+ __entry->vq = vq;
+ __entry->queue_index = vq->queue_index;
+ __entry->used_flags = vq->used_flags;
+ ),
+
+ TP_printk("vhost update used flag %x to vq %d notify %s",
+ __entry->used_flags, __entry->queue_index,
+ (__entry->used_flags & VRING_USED_F_NO_NOTIFY) ?
+ "disabled" : "enabled")
+);
+
+/*
+ * Tracepoint for updating avail event.
+ */
+TRACE_EVENT(vhost_virtio_update_avail_event,
+ TP_PROTO(struct vhost_virtqueue *vq),
+ TP_ARGS(vq),
+
+ TP_STRUCT__entry(
+ __field(struct vhost_virtqueue *, vq)
+ __field(u16, queue_index)
+ __field(u16, avail_idx)
+ ),
+
+ TP_fast_assign(
+ __entry->vq = vq;
+ __entry->queue_index = vq->queue_index;
+ __entry->avail_idx = vq->avail_idx;
+ ),
+
+ TP_printk("vhost update avail event %u(%u) for vq %d",
+ __entry->avail_idx, __entry->avail_idx %
+ __entry->vq->num, __entry->queue_index)
+);
+
+/*
+ * Tracepoint for updating used index.
+ */
+TRACE_EVENT(vhost_virtio_update_used_idx,
+ TP_PROTO(struct vhost_virtqueue *vq),
+ TP_ARGS(vq),
+
+ TP_STRUCT__entry(
+ __field(struct vhost_virtqueue *, vq)
+ __field(u16, queue_index)
+ __field(u16, used_idx)
+ ),
+
+ TP_fast_assign(
+ __entry->vq = vq;
+ __entry->queue_index = vq->queue_index;
+ __entry->used_idx = vq->last_used_idx;
+ ),
+
+ TP_printk("vhost update used index %u(%u) for vq %d",
+ __entry->used_idx, __entry->used_idx %
+ __entry->vq->num, __entry->queue_index)
+);
+
+/*
+ * Tracepoint for processing descriptor.
+ */
+TRACE_EVENT(vhost_virtio_get_vq_desc,
+ TP_PROTO(struct vhost_virtqueue *vq, unsigned int index,
+ unsigned out, unsigned int in),
+ TP_ARGS(vq, index, out, in),
+
+ TP_STRUCT__entry(
+ __field(struct vhost_virtqueue *, vq)
+ __field(u16, queue_index)
+ __field(unsigned int, head)
+ __field(unsigned int, out)
+ __field(unsigned int, in)
+ __field(u16, last_avail_idx)
+ ),
+
+ TP_fast_assign(
+ __entry->vq = vq;
+ __entry->queue_index = vq->queue_index;
+ __entry->head = index;
+ __entry->out = out;
+ __entry->in = in;
+ __entry->last_avail_idx = vq->last_avail_idx;
+ ),
+
+ TP_printk("vhost get vq %d desc last avail index %u(%u), "
+ "head %u out %u in %u",
+ __entry->queue_index,
+ __entry->last_avail_idx,
+ __entry->last_avail_idx % __entry->vq->num,
+ __entry->head, __entry->out, __entry->in)
+);
+
+/*
+ * Tracepoint for signal guest.
+ */
+TRACE_EVENT(vhost_virtio_signal,
+ TP_PROTO(struct vhost_virtqueue *vq),
+ TP_ARGS(vq),
+
+ TP_STRUCT__entry(
+ __field(u16, queue_index)
+ ),
+
+ TP_fast_assign(
+ __entry->queue_index = vq->queue_index;
+ ),
+
+ TP_printk("vhost signal vq %d", __entry->queue_index)
+);
+
+DECLARE_EVENT_CLASS(vhost_work_template,
+ TP_PROTO(struct vhost_work *work),
+ TP_ARGS(work),
+
+ TP_STRUCT__entry(
+ __field(void *, function)
+ ),
+
+ TP_fast_assign(
+ __entry->function = work->fn;
+ ),
+
+ TP_printk("%pf", __entry->function)
+);
+
+DEFINE_EVENT(vhost_work_template, vhost_work_queue_wakeup,
+ TP_PROTO(struct vhost_work *work),
+ TP_ARGS(work));
+
+DEFINE_EVENT(vhost_work_template, vhost_work_queue_coalesce,
+ TP_PROTO(struct vhost_work *work),
+ TP_ARGS(work));
+
+DEFINE_EVENT(vhost_work_template, vhost_poll_start,
+ TP_PROTO(struct vhost_work *work),
+ TP_ARGS(work));
+
+DEFINE_EVENT(vhost_work_template, vhost_poll_stop,
+ TP_PROTO(struct vhost_work *work),
+ TP_ARGS(work));
+
+#endif /* _TRACE_VHOST_H */
+
+#undef TRACE_INCLUDE_PATH
+#define TRACE_INCLUDE_PATH ../../drivers/vhost
+#undef TRACE_INCLUDE_FILE
+#define TRACE_INCLUDE_FILE trace
+
+/* This part must be outside protection */
+#include <trace/define_trace.h>
+
diff --git a/drivers/vhost/vhost.c b/drivers/vhost/vhost.c
index 78987e4..7cf3d6e 100644
--- a/drivers/vhost/vhost.c
+++ b/drivers/vhost/vhost.c
@@ -28,6 +28,8 @@
#include <linux/module.h>
#include "vhost.h"
+#define CREATE_TRACE_POINTS
+#include "trace.h"
enum {
VHOST_MEMORY_MAX_NREGIONS = 64,
@@ -45,6 +47,7 @@ static void vhost_poll_func(struct file *file,
wait_queue_head_t *wqh,
poll = container_of(pt, struct vhost_poll, table);
poll->wqh = wqh;
add_wait_queue(wqh, &poll->wait);
+ trace_vhost_poll_start(&poll->work);
}
static int vhost_poll_wakeup(wait_queue_t *wait, unsigned mode, int sync,
@@ -114,6 +117,7 @@ void vhost_poll_stop(struct vhost_poll *poll)
remove_wait_queue(poll->wqh, &poll->wait);
poll->wqh = NULL;
}
+ trace_vhost_poll_stop(&poll->work);
}
EXPORT_SYMBOL_GPL(vhost_poll_stop);
@@ -163,8 +167,10 @@ void vhost_work_queue(struct vhost_dev *dev, struct
vhost_work *work)
work->queue_seq++;
spin_unlock_irqrestore(&dev->work_lock, flags);
wake_up_process(dev->worker);
+ trace_vhost_work_queue_wakeup(work);
} else {
spin_unlock_irqrestore(&dev->work_lock, flags);
+ trace_vhost_work_queue_coalesce(work);
}
}
EXPORT_SYMBOL_GPL(vhost_work_queue);
@@ -1008,6 +1014,7 @@ static int vhost_update_used_flags(struct vhost_virtqueue
*vq)
if (vq->log_ctx)
eventfd_signal(vq->log_ctx, 1);
}
+ trace_vhost_virtio_update_used_flags(vq);
return 0;
}
@@ -1027,6 +1034,7 @@ static int vhost_update_avail_event(struct vhost_virtqueue
*vq, u16 avail_event)
if (vq->log_ctx)
eventfd_signal(vq->log_ctx, 1);
}
+ trace_vhost_virtio_update_avail_event(vq);
return 0;
}
@@ -1311,6 +1319,7 @@ int vhost_get_vq_desc(struct vhost_dev *dev, struct
vhost_virtqueue *vq,
}
} while ((i = next_desc(&desc)) != -1);
+ trace_vhost_virtio_get_vq_desc(vq, head, *out_num, *in_num);
/* On success, increment avail index. */
vq->last_avail_idx++;
@@ -1405,6 +1414,7 @@ int vhost_add_used_n(struct vhost_virtqueue *vq, struct
vring_used_elem *heads,
vq_err(vq, "Failed to increment used idx");
return -EFAULT;
}
+ trace_vhost_virtio_update_used_idx(vq);
if (unlikely(vq->log_used)) {
/* Log used index update. */
log_write(vq->log_base,
@@ -1457,8 +1467,10 @@ static bool vhost_notify(struct vhost_dev *dev, struct
vhost_virtqueue *vq)
void vhost_signal(struct vhost_dev *dev, struct vhost_virtqueue *vq)
{
/* Signal the Guest tell them we used something up. */
- if (vq->call_ctx && vhost_notify(dev, vq))
+ if (vq->call_ctx && vhost_notify(dev, vq)) {
eventfd_signal(vq->call_ctx, 1);
+ trace_vhost_virtio_signal(vq);
+ }
}
EXPORT_SYMBOL_GPL(vhost_signal);
--
1.8.3.2
Jason Wang
2014-Mar-21 09:41 UTC
[PATCH RFC V2 3/4] vhost_net: add basic tracepoints for vhost_net
To help performance analyze and debugging, this patch introduces
tracepoints for vhost_net. Two tracepoints were introduced, packets
sending and receiving.
Signed-off-by: Jason Wang <jasowang at redhat.com>
---
drivers/vhost/net.c | 5 +++++
drivers/vhost/net_trace.h | 53 +++++++++++++++++++++++++++++++++++++++++++++++
2 files changed, 58 insertions(+)
create mode 100644 drivers/vhost/net_trace.h
diff --git a/drivers/vhost/net.c b/drivers/vhost/net.c
index 7353204..8ac83a9 100644
--- a/drivers/vhost/net.c
+++ b/drivers/vhost/net.c
@@ -30,6 +30,9 @@
#include "vhost.h"
+#define CREATE_TRACE_POINTS
+#include "net_trace.h"
+
static int experimental_zcopytx = 1;
module_param(experimental_zcopytx, int, 0444);
MODULE_PARM_DESC(experimental_zcopytx, "Enable Zero Copy TX;"
@@ -444,6 +447,7 @@ static void handle_tx(struct vhost_net *net)
if (err != len)
pr_debug("Truncated TX packet: "
" len %d != %zd\n", err, len);
+ trace_vhost_net_tx(zcopy_used, len);
if (!zcopy_used)
vhost_add_used_and_signal(&net->dev, vq, head, 0);
else
@@ -620,6 +624,7 @@ static void handle_rx(struct vhost_net *net)
vhost_discard_vq_desc(vq, headcount);
continue;
}
+ trace_vhost_net_rx(sock_len);
if (unlikely(vhost_hlen) &&
memcpy_toiovecend(nvq->hdr, (unsigned char *)&hdr, 0,
vhost_hlen)) {
diff --git a/drivers/vhost/net_trace.h b/drivers/vhost/net_trace.h
new file mode 100644
index 0000000..a99a799
--- /dev/null
+++ b/drivers/vhost/net_trace.h
@@ -0,0 +1,53 @@
+#if !defined(_TRACE_VHOST_NET_H) || defined(TRACE_HEADER_MULTI_READ)
+#define _TRACE_VHOST_NET_H
+
+#include <linux/tracepoint.h>
+#include "vhost.h"
+
+#undef TRACE_SYSTEM
+#define TRACE_SYSTEM vhost_net
+
+TRACE_EVENT(vhost_net_tx,
+ TP_PROTO(bool zerocopy, int len),
+ TP_ARGS(zerocopy, len),
+
+ TP_STRUCT__entry(
+ __field(bool, zerocopy)
+ __field(int, len)
+ ),
+
+ TP_fast_assign(
+ __entry->zerocopy = zerocopy;
+ __entry->len = len;
+ ),
+
+ TP_printk("vhost_net send packet mode %s length %d\n",
+ __entry->zerocopy ? "zerocopy" : "datacopy",
+ __entry->len)
+);
+
+TRACE_EVENT(vhost_net_rx,
+ TP_PROTO(int len),
+ TP_ARGS(len),
+
+ TP_STRUCT__entry(
+ __field(int, len)
+ ),
+
+ TP_fast_assign(
+ __entry->len = len;
+ ),
+
+ TP_printk("vhost_net receive packet length %d\n",
+ __entry->len)
+);
+
+#endif /* _TRACE_VHOST_NET_H */
+
+#undef TRACE_INCLUDE_PATH
+#define TRACE_INCLUDE_PATH ../../drivers/vhost
+#undef TRACE_INCLUDE_FILE
+#define TRACE_INCLUDE_FILE net_trace
+
+/* This part must be outside protection */
+#include <trace/define_trace.h>
--
1.8.3.2
Jason Wang
2014-Mar-21 09:41 UTC
[PATCH RFC V2 4/4] tools: virtio: add a top-like utility for displaying vhost satistics
This patch adds simple python to display vhost satistics of vhost, the codes
were based on kvm_stat script from qemu. As work function has been recored,
filters could be used to distinguish which kinds of work are being executed or
queued:
vhost statistics
vhost_virtio_update_used_idx 1215215 0
vhost_virtio_get_vq_desc 1215215 0
vhost_work_queue_wakeup 986808 0
vhost_virtio_signal 811601 0
vhost_net_tx 611457 0
vhost_net_rx 603758 0
vhost_net_tx(datacopy) 601903 0
vhost_work_queue_wakeup(rx_net) 565081 0
vhost_virtio_signal(rx) 461603 0
vhost_work_queue_wakeup(tx_kick) 421718 0
vhost_virtio_update_avail_event 417346 0
vhost_virtio_signal(tx) 349998 0
vhost_work_queue_coalesce 39384 0
vhost_work_queue_coalesce(rx_net) 38677 0
vhost_net_tx(zerocopy) 9554 0
vhost_work_queue_coalesce(tx_kick) 707 0
vhost_work_queue_wakeup(rx_kick) 9 0
Signed-off-by: Jason Wang <jasowang at redhat.com>
---
tools/virtio/vhost_stat | 375 ++++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 375 insertions(+)
create mode 100755 tools/virtio/vhost_stat
diff --git a/tools/virtio/vhost_stat b/tools/virtio/vhost_stat
new file mode 100755
index 0000000..398fd4a
--- /dev/null
+++ b/tools/virtio/vhost_stat
@@ -0,0 +1,375 @@
+#!/usr/bin/python
+#
+# top-like utility for displaying vhost statistics
+#
+# Copyright 2012 Red Hat, Inc.
+#
+# Modified from kvm_stat from qemu
+#
+# This work is licensed under the terms of the GNU GPL, version 2. See
+# the COPYING file in the top-level directory.
+
+import curses
+import sys, os, time, optparse
+
+work_types = {
+ "handle_rx_kick" : "rx_kick",
+ "handle_tx_kick" : "tx_kick",
+ "handle_rx_net" : "rx_net",
+ "handle_tx_net" : "tx_net",
+ "vhost_attach_cgroups_work": "cg_attach"
+ }
+
+addr = {}
+
+kallsyms = file("/proc/kallsyms").readlines()
+for kallsym in kallsyms:
+ entry = kallsym.split()
+ if entry[2] in work_types.keys():
+ addr["0x%s" % entry[0]] = work_types[entry[2]]
+
+copy_modes = {
+ 1 : 'zerocopy',
+ 0 : 'datacopy',
+}
+
+vqs = {
+ 1 : 'rx',
+ 0 : 'tx',
+}
+
+filters = {
+ 'vhost_work_queue_wakeup': ('function', addr),
+ 'vhost_work_queue_coalesce' : ('function', addr),
+ 'vhost_poll_start' : ('function', addr),
+ 'vhost_poll_stop' : ('function', addr),
+ 'vhost_net_tx' : ('zerocopy', copy_modes),
+ 'vhost_virtio_signal' : ('queue_index', vqs),
+}
+
+def invert(d):
+ return dict((x[1], x[0]) for x in d.iteritems())
+
+for f in filters:
+ filters[f] = (filters[f][0], invert(filters[f][1]))
+
+import ctypes, struct, array
+
+libc = ctypes.CDLL('libc.so.6')
+syscall = libc.syscall
+class perf_event_attr(ctypes.Structure):
+ _fields_ = [('type', ctypes.c_uint32),
+ ('size', ctypes.c_uint32),
+ ('config', ctypes.c_uint64),
+ ('sample_freq', ctypes.c_uint64),
+ ('sample_type', ctypes.c_uint64),
+ ('read_format', ctypes.c_uint64),
+ ('flags', ctypes.c_uint64),
+ ('wakeup_events', ctypes.c_uint32),
+ ('bp_type', ctypes.c_uint32),
+ ('bp_addr', ctypes.c_uint64),
+ ('bp_len', ctypes.c_uint64),
+ ]
+def _perf_event_open(attr, pid, cpu, group_fd, flags):
+ return syscall(298, ctypes.pointer(attr), ctypes.c_int(pid),
+ ctypes.c_int(cpu), ctypes.c_int(group_fd),
+ ctypes.c_long(flags))
+
+PERF_TYPE_HARDWARE = 0
+PERF_TYPE_SOFTWARE = 1
+PERF_TYPE_TRACEPOINT = 2
+PERF_TYPE_HW_CACHE = 3
+PERF_TYPE_RAW = 4
+PERF_TYPE_BREAKPOINT = 5
+
+PERF_SAMPLE_IP = 1 << 0
+PERF_SAMPLE_TID = 1 << 1
+PERF_SAMPLE_TIME = 1 << 2
+PERF_SAMPLE_ADDR = 1 << 3
+PERF_SAMPLE_READ = 1 << 4
+PERF_SAMPLE_CALLCHAIN = 1 << 5
+PERF_SAMPLE_ID = 1 << 6
+PERF_SAMPLE_CPU = 1 << 7
+PERF_SAMPLE_PERIOD = 1 << 8
+PERF_SAMPLE_STREAM_ID = 1 << 9
+PERF_SAMPLE_RAW = 1 << 10
+
+PERF_FORMAT_TOTAL_TIME_ENABLED = 1 << 0
+PERF_FORMAT_TOTAL_TIME_RUNNING = 1 << 1
+PERF_FORMAT_ID = 1 << 2
+PERF_FORMAT_GROUP = 1 << 3
+
+import re
+
+sys_tracing = '/sys/kernel/debug/tracing'
+
+class Group(object):
+ def __init__(self, cpu):
+ self.events = []
+ self.group_leader = None
+ self.cpu = cpu
+ def add_event(self, name, event_set, tracepoint, filter = None):
+ self.events.append(Event(group = self,
+ name = name, event_set = event_set,
+ tracepoint = tracepoint, filter = filter))
+ if len(self.events) == 1:
+ self.file = os.fdopen(self.events[0].fd)
+ def read(self):
+ bytes = 8 * (1 + len(self.events))
+ fmt = 'xxxxxxxx' + 'q' * len(self.events)
+ return dict(zip([event.name for event in self.events],
+ struct.unpack(fmt, self.file.read(bytes))))
+
+class Event(object):
+ def __init__(self, group, name, event_set, tracepoint, filter = None):
+ self.name = name
+ attr = perf_event_attr()
+ attr.type = PERF_TYPE_TRACEPOINT
+ attr.size = ctypes.sizeof(attr)
+ id_path = os.path.join(sys_tracing, 'events', event_set,
+ tracepoint, 'id')
+ id = int(file(id_path).read())
+ attr.config = id
+ attr.sample_type = (PERF_SAMPLE_RAW
+ | PERF_SAMPLE_TIME
+ | PERF_SAMPLE_CPU)
+ attr.sample_period = 1
+ attr.read_format = PERF_FORMAT_GROUP
+ group_leader = -1
+ if group.events:
+ group_leader = group.events[0].fd
+ fd = _perf_event_open(attr, -1, group.cpu, group_leader, 0)
+ if fd == -1:
+ raise Exception('perf_event_open failed')
+ if filter:
+ import fcntl
+ fcntl.ioctl(fd, 0x40082406, filter)
+ self.fd = fd
+ def enable(self):
+ import fcntl
+ fcntl.ioctl(self.fd, 0x00002400, 0)
+ def disable(self):
+ import fcntl
+ fcntl.ioctl(self.fd, 0x00002401, 0)
+
+class TracepointProvider(object):
+ def __init__(self, event_set):
+ path = os.path.join(sys_tracing, 'events', event_set)
+ self.event_set = event_set
+ fields = [f
+ for f in os.listdir(path)
+ if os.path.isdir(os.path.join(path, f))]
+ extra = []
+ for f in fields:
+ if f in filters:
+ subfield, values = filters[f]
+ for name, number in values.iteritems():
+ # kvm_exit(MMIO)
+ extra.append(f + '(' + name + ')')
+ fields += extra
+ self._setup(fields)
+ self.select(fields)
+ def fields(self):
+ return self._fields
+ def _setup(self, _fields):
+ self._fields = _fields
+ cpure = r'cpu([0-9]+)'
+ self.cpus = [int(re.match(cpure, x).group(1))
+ for x in os.listdir('/sys/devices/system/cpu')
+ if re.match(cpure, x)]
+ import resource
+ nfiles = len(self.cpus) * 1000
+ resource.setrlimit(resource.RLIMIT_NOFILE, (nfiles, nfiles))
+ events = []
+ self.group_leaders = []
+ for cpu in self.cpus:
+ group = Group(cpu)
+ for name in _fields:
+ tracepoint = name
+ filter = None
+ # for field like kvm_exit(MMIO)
+ m = re.match(r'(.*)\((.*)\)', name)
+ if m:
+ tracepoint, sub = m.groups()
+ filter = '%s==%s\0' % (filters[tracepoint][0],
+ filters[tracepoint][1][sub])
+ event = group.add_event(name, event_set = self.event_set,
+ tracepoint = tracepoint,
+ filter = filter)
+ self.group_leaders.append(group)
+ def select(self, fields):
+ for group in self.group_leaders:
+ for event in group.events:
+ if event.name in fields:
+ event.enable()
+ else:
+ event.disable()
+ def read(self):
+ from collections import defaultdict
+ ret = defaultdict(int)
+ for group in self.group_leaders:
+ for name, val in group.read().iteritems():
+ ret[name] += val
+ return ret
+
+class Stats:
+ def __init__(self, providers, fields = None):
+ self.providers = providers
+ self.fields_filter = fields
+ self._update()
+ def _update(self):
+ def wanted(key):
+ import re
+ if not self.fields_filter:
+ return True
+ return re.match(self.fields_filter, key) is not None
+ self.values = {}
+ for provider in self.providers:
+ for key in provider.fields():
+ if wanted(key):
+ self.values[key] = None
+ provider.select(self.values.keys())
+ def set_fields_filter(self, fields_filter):
+ self.fields_filter = fields_filter
+ self._update()
+ def get(self):
+ for provider in self.providers:
+ new = provider.read()
+ for key in provider.fields():
+ oldval = self.values.get(key, (0, 0))
+ newval = new[key]
+ newdelta = None
+ if oldval is not None:
+ newdelta = newval - oldval[0]
+ self.values[key] = (newval, newdelta)
+ return self.values
+
+if not os.access('/sys/kernel/debug', os.F_OK):
+ print 'Please enable CONFIG_DEBUG_FS in your kernel'
+ sys.exit(1)
+if not os.access('/sys/module/vhost_net', os.F_OK):
+ print 'Please make sure vhost_net module are loaded'
+ sys.exit(1)
+
+label_width = 40
+number_width = 10
+
+def tui(screen, stats):
+ curses.use_default_colors()
+ curses.noecho()
+ drilldown = False
+ fields_filter = stats.fields_filter
+ def update_drilldown():
+ if not fields_filter:
+ if drilldown:
+ stats.set_fields_filter(None)
+ else:
+ stats.set_fields_filter(r'^[^\(]*$')
+ update_drilldown()
+ def refresh(sleeptime):
+ screen.erase()
+ screen.addstr(0, 0, 'vhost statistics')
+ row = 2
+ s = stats.get()
+ def sortkey(x):
+ if s[x][1]:
+ return (-s[x][1], -s[x][0])
+ else:
+ return (0, -s[x][0])
+ for key in sorted(s.keys(), key = sortkey):
+ if row >= screen.getmaxyx()[0]:
+ break
+ values = s[key]
+ if not values[0] and not values[1]:
+ break
+ col = 1
+ screen.addstr(row, col, key)
+ col += label_width
+ screen.addstr(row, col, '%10d' % (values[0],))
+ col += number_width
+ if values[1] is not None:
+ screen.addstr(row, col, '%8d' % (values[1] /
sleeptime,))
+ row += 1
+ screen.refresh()
+
+ sleeptime = 0.25
+ while True:
+ refresh(sleeptime)
+ curses.halfdelay(int(sleeptime * 10))
+ sleeptime = 3
+ try:
+ c = screen.getkey()
+ if c == 'x':
+ drilldown = not drilldown
+ update_drilldown()
+ if c == 'q':
+ break
+ except KeyboardInterrupt:
+ break
+ except curses.error:
+ continue
+
+def batch(stats):
+ s = stats.get()
+ time.sleep(1)
+ s = stats.get()
+ for key in sorted(s.keys()):
+ values = s[key]
+ print '%-22s%10d%10d' % (key, values[0], values[1])
+
+def log(stats):
+ keys = sorted(stats.get().iterkeys())
+ def banner():
+ for k in keys:
+ print '%10s' % k[0:9],
+ print
+ def statline():
+ s = stats.get()
+ for k in keys:
+ print ' %9d' % s[k][1],
+ print
+ line = 0
+ banner_repeat = 20
+ while True:
+ time.sleep(1)
+ if line % banner_repeat == 0:
+ banner()
+ statline()
+ line += 1
+
+options = optparse.OptionParser()
+options.add_option('-1', '--once', '--batch',
+ action = 'store_true',
+ default = False,
+ dest = 'once',
+ help = 'run in batch mode for one second',
+ )
+options.add_option('-l', '--log',
+ action = 'store_true',
+ default = False,
+ dest = 'log',
+ help = 'run in logging mode (like vmstat)',
+ )
+options.add_option('-f', '--fields',
+ action = 'store',
+ default = None,
+ dest = 'fields',
+ help = 'fields to display (regex)',
+ )
+(options, args) = options.parse_args(sys.argv)
+
+try:
+ provider = [TracepointProvider('vhost'),
TracepointProvider('vhost_net')]
+except:
+ print "Could not initialize tracepoint"
+ sys.exit(1)
+
+stats = Stats(provider, fields = options.fields)
+
+if options.log:
+ log(stats)
+elif not options.once:
+ import curses.wrapper
+ curses.wrapper(tui, stats)
+else:
+ batch(stats)
--
1.8.3.2
Fam Zheng
2014-Apr-10 09:27 UTC
[PATCH RFC V2 4/4] tools: virtio: add a top-like utility for displaying vhost satistics
On Fri, 03/21 17:41, Jason Wang wrote:> This patch adds simple python to display vhost satistics of vhost, the codes > were based on kvm_stat script from qemu. As work function has been recored, > filters could be used to distinguish which kinds of work are being executed or > queued: > > vhost statistics > > vhost_virtio_update_used_idx 1215215 0 > vhost_virtio_get_vq_desc 1215215 0 > vhost_work_queue_wakeup 986808 0 > vhost_virtio_signal 811601 0 > vhost_net_tx 611457 0 > vhost_net_rx 603758 0 > vhost_net_tx(datacopy) 601903 0 > vhost_work_queue_wakeup(rx_net) 565081 0 > vhost_virtio_signal(rx) 461603 0 > vhost_work_queue_wakeup(tx_kick) 421718 0 > vhost_virtio_update_avail_event 417346 0 > vhost_virtio_signal(tx) 349998 0 > vhost_work_queue_coalesce 39384 0 > vhost_work_queue_coalesce(rx_net) 38677 0 > vhost_net_tx(zerocopy) 9554 0 > vhost_work_queue_coalesce(tx_kick) 707 0 > vhost_work_queue_wakeup(rx_kick) 9 0 > > Signed-off-by: Jason Wang <jasowang at redhat.com> > --- > tools/virtio/vhost_stat | 375 ++++++++++++++++++++++++++++++++++++++++++++++++ > 1 file changed, 375 insertions(+) > create mode 100755 tools/virtio/vhost_stat > > diff --git a/tools/virtio/vhost_stat b/tools/virtio/vhost_stat > new file mode 100755 > index 0000000..398fd4a > --- /dev/null > +++ b/tools/virtio/vhost_stat > @@ -0,0 +1,375 @@ > +#!/usr/bin/python > +# > +# top-like utility for displaying vhost statistics > +# > +# Copyright 2012 Red Hat, Inc.Should it be 2014? Fam <snip>
Apparently Analagous Threads
- [PATCH RFC V2 0/4] Adding tracepoints to vhost/net
- [PATCH RFC V2 0/4] Adding tracepoints to vhost/net
- [PATCH 0/2] adding tracepoints to vhost
- [PATCH 0/2] adding tracepoints to vhost
- [PATCH RFC V2 4/4] tools: virtio: add a top-like utility for displaying vhost satistics