Hi,I am having an issue on several KVM hosts which run CentOs7
and libvirt-daemon-1.2.17-13.el7_2.3.x86_64 .About once a week the libvirtd
hangs forever and becomes unresponsive, the only help is to restart it. I
attached gdb to it and the output of it can be seen at the end of the mail.
Laine from the irc channel suggests it has to do with dbus/policykit. If anybody
got an idea what it could be and point me in the right direction or help further
analyzing this issue I'd be very grateful.
kind regards
Philippe d'Anjou
(gdb) thread apply all bt
Thread 16 (Thread 0x7f275e916700 (LWP 1973)):
#0 0x00007f276b5a06d5 in pthread_cond_wait@@GLIBC_2.3.2 () from
/lib64/libpthread.so.0
#1 0x00007f276df31c86 in virCondWait () from /lib64/libvirt.so.0
#2 0x00007f276df3258b in virThreadPoolWorker () from /lib64/libvirt.so.0
#3 0x00007f276df31a18 in virThreadHelper () from /lib64/libvirt.so.0
#4 0x00007f276b59cdc5 in start_thread () from /lib64/libpthread.so.0
#5 0x00007f276b2ca28d in clone () from /lib64/libc.so.6
Thread 15 (Thread 0x7f275e115700 (LWP 1974)):
#0 0x00007f276b5a06d5 in pthread_cond_wait@@GLIBC_2.3.2 () from
/lib64/libpthread.so.0
#1 0x00007f276df31c86 in virCondWait () from /lib64/libvirt.so.0
#2 0x00007f276df3258b in virThreadPoolWorker () from /lib64/libvirt.so.0
#3 0x00007f276df31a18 in virThreadHelper () from /lib64/libvirt.so.0
#4 0x00007f276b59cdc5 in start_thread () from /lib64/libpthread.so.0
#5 0x00007f276b2ca28d in clone () from /lib64/libc.so.6
Thread 14 (Thread 0x7f275d914700 (LWP 1975)):
#0 0x00007f276b5a06d5 in pthread_cond_wait@@GLIBC_2.3.2 () from
/lib64/libpthread.so.0
#1 0x00007f276df31c86 in virCondWait () from /lib64/libvirt.so.0
#2 0x00007f276df3258b in virThreadPoolWorker () from /lib64/libvirt.so.0
#3 0x00007f276df31a18 in virThreadHelper () from /lib64/libvirt.so.0
#4 0x00007f276b59cdc5 in start_thread () from /lib64/libpthread.so.0
#5 0x00007f276b2ca28d in clone () from /lib64/libc.so.6
Thread 13 (Thread 0x7f275d113700 (LWP 1976)):
#0 0x00007f276b5a06d5 in pthread_cond_wait@@GLIBC_2.3.2 () from
/lib64/libpthread.so.0
#1 0x00007f276df31c86 in virCondWait () from /lib64/libvirt.so.0
#2 0x00007f276df3258b in virThreadPoolWorker () from /lib64/libvirt.so.0
#3 0x00007f276df31a18 in virThreadHelper () from /lib64/libvirt.so.0
#4 0x00007f276b59cdc5 in start_thread () from /lib64/libpthread.so.0
#5 0x00007f276b2ca28d in clone () from /lib64/libc.so.6
Thread 12 (Thread 0x7f275c912700 (LWP 1977)):
#0 0x00007f276b5a06d5 in pthread_cond_wait@@GLIBC_2.3.2 () from
/lib64/libpthread.so.0
#1 0x00007f276df31c86 in virCondWait () from /lib64/libvirt.so.0
#2 0x00007f276df3258b in virThreadPoolWorker () from /lib64/libvirt.so.0
#3 0x00007f276df31a18 in virThreadHelper () from /lib64/libvirt.so.0
#4 0x00007f276b59cdc5 in start_thread () from /lib64/libpthread.so.0
#5 0x00007f276b2ca28d in clone () from /lib64/libc.so.6
Thread 11 (Thread 0x7f275c111700 (LWP 1978)):
#0 0x00007f276b5a06d5 in pthread_cond_wait@@GLIBC_2.3.2 () from
/lib64/libpthread.so.0
#1 0x00007f276df31c86 in virCondWait () from /lib64/libvirt.so.0
#2 0x00007f276df325ab in virThreadPoolWorker () from /lib64/libvirt.so.0
#3 0x00007f276df31a18 in virThreadHelper () from /lib64/libvirt.so.0
#4 0x00007f276b59cdc5 in start_thread () from /lib64/libpthread.so.0
---Type <return> to continue, or q <return> to quit---
#5 0x00007f276b2ca28d in clone () from /lib64/libc.so.6
Thread 10 (Thread 0x7f275b910700 (LWP 1979)):
#0 0x00007f276b5a06d5 in pthread_cond_wait@@GLIBC_2.3.2 () from
/lib64/libpthread.so.0
#1 0x00007f276df31c86 in virCondWait () from /lib64/libvirt.so.0
#2 0x00007f276df325ab in virThreadPoolWorker () from /lib64/libvirt.so.0
#3 0x00007f276df31a18 in virThreadHelper () from /lib64/libvirt.so.0
#4 0x00007f276b59cdc5 in start_thread () from /lib64/libpthread.so.0
#5 0x00007f276b2ca28d in clone () from /lib64/libc.so.6
Thread 9 (Thread 0x7f275b10f700 (LWP 1980)):
#0 0x00007f276b2bfc3d in poll () from /lib64/libc.so.6
#1 0x00007f276c5bc770 in socket_do_iteration () from /lib64/libdbus-1.so.3
#2 0x00007f276c5bb5ef in _dbus_transport_do_iteration () from
/lib64/libdbus-1.so.3
#3 0x00007f276c5a4d7c in _dbus_connection_do_iteration_unlocked () from
/lib64/libdbus-1.so.3
#4 0x00007f276c5a583c in _dbus_connection_block_pending_call () from
/lib64/libdbus-1.so.3
#5 0x00007f276c5a5d3a in dbus_connection_send_with_reply_and_block () from
/lib64/libdbus-1.so.3
#6 0x00007f276deea258 in virDBusCallMethod () from /lib64/libvirt.so.0
#7 0x00007f276df1fd14 in virPolkitCheckAuth () from /lib64/libvirt.so.0
#8 0x00007f276ec1d83a in remoteDispatchAuthPolkitHelper ()
#9 0x00007f276e03c4e2 in virNetServerProgramDispatch () from
/lib64/libvirt.so.0
#10 0x00007f276e03775d in virNetServerHandleJob () from /lib64/libvirt.so.0
#11 0x00007f276df324f5 in virThreadPoolWorker () from /lib64/libvirt.so.0
#12 0x00007f276df31a18 in virThreadHelper () from /lib64/libvirt.so.0
#13 0x00007f276b59cdc5 in start_thread () from /lib64/libpthread.so.0
#14 0x00007f276b2ca28d in clone () from /lib64/libc.so.6
Thread 8 (Thread 0x7f275a90e700 (LWP 1981)):
#0 0x00007f276b5a06d5 in pthread_cond_wait@@GLIBC_2.3.2 () from
/lib64/libpthread.so.0
#1 0x00007f276df31c86 in virCondWait () from /lib64/libvirt.so.0
#2 0x00007f276df325ab in virThreadPoolWorker () from /lib64/libvirt.so.0
#3 0x00007f276df31a18 in virThreadHelper () from /lib64/libvirt.so.0
#4 0x00007f276b59cdc5 in start_thread () from /lib64/libpthread.so.0
#5 0x00007f276b2ca28d in clone () from /lib64/libc.so.6
Thread 7 (Thread 0x7f275a10d700 (LWP 1982)):
#0 0x00007f276b5a06d5 in pthread_cond_wait@@GLIBC_2.3.2 () from
/lib64/libpthread.so.0
#1 0x00007f276df31c86 in virCondWait () from /lib64/libvirt.so.0
#2 0x00007f276df325ab in virThreadPoolWorker () from /lib64/libvirt.so.0
#3 0x00007f276df31a18 in virThreadHelper () from /lib64/libvirt.so.0
#4 0x00007f276b59cdc5 in start_thread () from /lib64/libpthread.so.0
#5 0x00007f276b2ca28d in clone () from /lib64/libc.so.6
Thread 6 (Thread 0x7f2754c89700 (LWP 1985)):
#0 0x00007f276b5a06d5 in pthread_cond_wait@@GLIBC_2.3.2 () from
/lib64/libpthread.so.0
#1 0x00007f276df31c86 in virCondWait () from /lib64/libvirt.so.0
#2 0x00007f276df3258b in virThreadPoolWorker () from /lib64/libvirt.so.0
---Type <return> to continue, or q <return> to quit---
#3 0x00007f276df31a18 in virThreadHelper () from /lib64/libvirt.so.0
#4 0x00007f276b59cdc5 in start_thread () from /lib64/libpthread.so.0
#5 0x00007f276b2ca28d in clone () from /lib64/libc.so.6
Thread 5 (Thread 0x7f2754488700 (LWP 1986)):
#0 0x00007f276b5a06d5 in pthread_cond_wait@@GLIBC_2.3.2 () from
/lib64/libpthread.so.0
#1 0x00007f276df31c86 in virCondWait () from /lib64/libvirt.so.0
#2 0x00007f276df3258b in virThreadPoolWorker () from /lib64/libvirt.so.0
#3 0x00007f276df31a18 in virThreadHelper () from /lib64/libvirt.so.0
#4 0x00007f276b59cdc5 in start_thread () from /lib64/libpthread.so.0
#5 0x00007f276b2ca28d in clone () from /lib64/libc.so.6
Thread 4 (Thread 0x7f2753c87700 (LWP 1987)):
#0 0x00007f276b5a06d5 in pthread_cond_wait@@GLIBC_2.3.2 () from
/lib64/libpthread.so.0
#1 0x00007f276df31c86 in virCondWait () from /lib64/libvirt.so.0
#2 0x00007f276df3258b in virThreadPoolWorker () from /lib64/libvirt.so.0
#3 0x00007f276df31a18 in virThreadHelper () from /lib64/libvirt.so.0
#4 0x00007f276b59cdc5 in start_thread () from /lib64/libpthread.so.0
#5 0x00007f276b2ca28d in clone () from /lib64/libc.so.6
Thread 3 (Thread 0x7f2753486700 (LWP 1988)):
#0 0x00007f276b5a06d5 in pthread_cond_wait@@GLIBC_2.3.2 () from
/lib64/libpthread.so.0
#1 0x00007f276df31c86 in virCondWait () from /lib64/libvirt.so.0
#2 0x00007f276df3258b in virThreadPoolWorker () from /lib64/libvirt.so.0
#3 0x00007f276df31a18 in virThreadHelper () from /lib64/libvirt.so.0
#4 0x00007f276b59cdc5 in start_thread () from /lib64/libpthread.so.0
#5 0x00007f276b2ca28d in clone () from /lib64/libc.so.6
Thread 2 (Thread 0x7f2752c85700 (LWP 1989)):
#0 0x00007f276b5a06d5 in pthread_cond_wait@@GLIBC_2.3.2 () from
/lib64/libpthread.so.0
#1 0x00007f276df31c86 in virCondWait () from /lib64/libvirt.so.0
#2 0x00007f276df3258b in virThreadPoolWorker () from /lib64/libvirt.so.0
#3 0x00007f276df31a18 in virThreadHelper () from /lib64/libvirt.so.0
#4 0x00007f276b59cdc5 in start_thread () from /lib64/libpthread.so.0
#5 0x00007f276b2ca28d in clone () from /lib64/libc.so.6
Thread 1 (Thread 0x7f276eb7a8c0 (LWP 1971)):
#0 0x00007f276b2bfc3d in poll () from /lib64/libc.so.6
#1 0x00007f276def0556 in virEventPollRunOnce () from /lib64/libvirt.so.0
#2 0x00007f276deef032 in virEventRunDefaultImpl () from /lib64/libvirt.so.0
#3 0x00007f276e037035 in virNetDaemonRun () from /lib64/libvirt.so.0
#4 0x00007f276ebfc524 in main ()
