ABHISHEK PALIWAL
2018-Sep-21 10:37 UTC
[Gluster-users] [Gluster-devel] Crash in glusterfs!!!
Hi Sanju,
Output of 't a a bt full'
(gdb) t a a bt full
Thread 7 (LWP 1743):
#0 0x00003fffa3ea7e88 in __lll_lock_wait (futex=0x0, private=0) at
lowlevellock.c:43
r4 = 128
r7 = 0
arg2 = 128
r5 = 2
r8 = 1
r0 = 221
r3 = 0
r6 = 0
arg1 = 0
__err = 221
__ret = 0
#1 0x00003fffa3e9ef64 in __GI___pthread_mutex_lock (mutex=0x100272a8) at
../nptl/pthread_mutex_lock.c:81
__futex = 0x100272a8
__PRETTY_FUNCTION__ = "__pthread_mutex_lock"
type = <optimized out>
id = <optimized out>
#2 0x00003fffa3f6ce8c in _gf_msg (domain=0x3fff98006c90
"c_glusterfs-client-0", file=0x3fff9fb34de0 "client.c",
function=0x3fff9fb34cd8 <__FUNCTION__.18849> "notify",
line=<optimized out>, level=<optimized out>,
errnum=<optimized out>,
trace=<optimized out>, msgid=114020,
fmt=0x3fff9fb35350 "parent translators are ready, attempting connect on
transport") at logging.c:2058
ret = <optimized out>
msgstr = <optimized out>
ap = <optimized out>
this = 0x3fff980061f0
ctx = 0x10027010
callstr = '\000' <repeats 4095 times>
passcallstr = 0
log_inited = 0
__PRETTY_FUNCTION__ = "_gf_msg"
#3 0x00003fff9fb084ac in notify (this=0x3fff980061f0, event=<optimized
out>, data=0x3fff98008c50) at client.c:2116
conf = 0x3fff98056dd0
__FUNCTION__ = "notify"
#4 0x00003fffa3f68ca0 in xlator_notify (xl=0x3fff980061f0,
event=<optimized out>, data=<optimized out>) at xlator.c:491
old_THIS = 0x3fff98008c50
ret = 0
#5 0x00003fffa3f87700 in default_notify (this=0x3fff98008c50,
event=<optimized out>, data=<optimized out>) at defaults.c:2302
list = 0x3fff9800a340
#6 0x00003fff9fac922c in afr_notify (this=0x3fff98008c50, event=1,
data=0x3fff9800a4c0, data2=<optimized out>) at afr-common.c:3967
priv = 0x3fff98010050
i = <optimized out>
up_children = <optimized out>
down_children = <optimized out>
propagate = 1
had_heard_from_all = <optimized out>
---Type <return> to continue, or q <return> to quit---
have_heard_from_all = 0
idx = <optimized out>
ret = 0
call_psh = <optimized out>
input = 0x0
output = 0x0
had_quorum = <optimized out>
has_quorum = <optimized out>
__FUNCTION__ = "afr_notify"
#7 0x00003fff9fad4994 in notify (this=<optimized out>,
event=<optimized
out>, data=<optimized out>) at afr.c:38
ret = -1
ap = 0x3fffa034cc58 ""
data2 = <optimized out>
#8 0x00003fffa3f68ca0 in xlator_notify (xl=0x3fff98008c50,
event=<optimized out>, data=<optimized out>) at xlator.c:491
old_THIS = 0x3fff9800a4c0
ret = 0
#9 0x00003fffa3f87700 in default_notify (this=0x3fff9800a4c0,
event=<optimized out>, data=<optimized out>) at defaults.c:2302
list = 0x3fff9800b710
#10 0x00003fff9fa6b1e4 in notify (this=<optimized out>,
event=<optimized
out>, data=<optimized out>) at io-stats.c:3064
ret = 0
args = {type = IOS_DUMP_TYPE_NONE, u = {logfp = 0x0, dict = 0x0}}
op = 0
list_cnt = 0
throughput = 0
time = 0
is_peek = _gf_false
ap = 0x3fffa034ce68 ""
__FUNCTION__ = "notify"
#11 0x00003fffa3f68ca0 in xlator_notify (xl=0x3fff9800a4c0,
event=<optimized out>, data=<optimized out>) at xlator.c:491
old_THIS = 0x3fffa402d290 <global_xlator>
ret = 0
#12 0x00003fffa3fbd560 in glusterfs_graph_parent_up (graph=<optimized
out>)
at graph.c:440
trav = 0x3fff9800a4c0
ret = <optimized out>
#13 0x00003fffa3fbdb90 in glusterfs_graph_activate (graph=0x3fff98000af0,
ctx=0x10027010) at graph.c:688
ret = <optimized out>
__FUNCTION__ = "glusterfs_graph_activate"
#14 0x000000001000a49c in glusterfs_process_volfp (ctx=0x10027010,
fp=0x3fff98001cd0) at glusterfsd.c:2221
graph = 0x3fff98000af0
ret = <optimized out>
trav = <optimized out>
__FUNCTION__ = <error reading variable __FUNCTION__ (Cannot access
memory at address 0x10010ec0)>
#15 0x000000001000fd08 in mgmt_getspec_cbk (req=<optimized out>,
iov=<optimized out>, count=<optimized out>, myframe=0x3fffa2bea06c)
at
glusterfsd-mgmt.c:1561
rsp = {op_ret = 1059, op_errno = 0,
spec = 0x3fff980018a0 "volume c_glusterfs-client-0\n type
protocol/client\n option password
5fd8d83d-99f3-4630-97c6-965d7a8ead62\n option username
e65687aa-e135-445e-8778-48bb8fb19640\n option transport-type tcp\n
"..., xdata = {xdata_len = 0, xdata_val = 0x0}}
---Type <return> to continue, or q <return> to quit---
frame = 0x3fffa2bea06c
ctx = 0x10027010
ret = <optimized out>
size = 1059
tmpfp = 0x3fff98001cd0
volfilebuf = 0x0
__FUNCTION__ = <error reading variable __FUNCTION__ (Cannot access
memory at address 0x10013570)>
#16 0x00003fffa3f21ec4 in rpc_clnt_handle_reply (clnt=0x10089020,
pollin=0x3fff98001760) at rpc-clnt.c:775
conn = 0x10089050
saved_frame = <optimized out>
ret = <optimized out>
req = 0x1008931c
xid = 1
__FUNCTION__ = "rpc_clnt_handle_reply"
#17 0x00003fffa3f223d0 in rpc_clnt_notify (trans=<optimized out>,
mydata=0x10089050, event=<optimized out>, data=<optimized out>) at
rpc-clnt.c:933
conn = 0x10089050
clnt = <optimized out>
ret = -1
req_info = 0x0
pollin = <optimized out>
clnt_mydata = 0x0
old_THIS = 0x3fffa402d290 <global_xlator>
__FUNCTION__ = "rpc_clnt_notify"
#18 0x00003fffa3f1d4fc in rpc_transport_notify (this=<optimized out>,
event=<optimized out>, data=<optimized out>) at rpc-transport.c:546
ret = -1
__FUNCTION__ = "rpc_transport_notify"
#19 0x00003fffa0401d44 in socket_event_poll_in (this=this at entry=0x1008ab80)
at socket.c:2236
ret = <optimized out>
pollin = 0x3fff98001760
priv = 0x1008b820
#20 0x00003fffa040489c in socket_event_handler (fd=<optimized out>,
idx=<optimized out>, data=0x1008ab80, poll_in=<optimized out>,
poll_out=<optimized out>, poll_err=<optimized out>)
at socket.c:2349
this = 0x1008ab80
priv = 0x1008b820
ret = <optimized out>
__FUNCTION__ = "socket_event_handler"
#21 0x00003fffa3fe2874 in event_dispatch_epoll_handler
(event=0x3fffa034d6a0, event_pool=0x10045bc0) at event-epoll.c:575
handler = @0x3fffa041f620: 0x3fffa04046f0 <socket_event_handler>
gen = 1
slot = 0x1007cd80
data = <optimized out>
ret = -1
fd = 9
ev_data = 0x3fffa034d6a8
idx = 1
#22 event_dispatch_epoll_worker (data=0x1008bd50) at event-epoll.c:678
---Type <return> to continue, or q <return> to quit---
event = {events = 1, data = {ptr = 0x100000001, fd = 1, u32 = 1,
u64 = 4294967297}}
ret = <optimized out>
ev_data = 0x1008bd50
event_pool = 0x10045bc0
myindex = <optimized out>
timetodie = 0
__FUNCTION__ = "event_dispatch_epoll_worker"
#23 0x00003fffa3e9bb30 in start_thread (arg=0x3fffa034e160) at
pthread_create.c:462
pd = 0x3fffa034e160
now = <optimized out>
unwind_buf = {cancel_jmp_buf = {{jmp_buf = {2380233324717718430,
70367199403008, 2380233324703897146, 0, 0, 70367128645632, 70367137030688,
8388608, 70367199363104, 269008208,
70368094386592, 70367199388632, 70367200825640, 3, 0,
70367199388648, 70368094386240, 70368094386296, 4001536, 70367199364120,
70367137027904, -3187653596,
0 <repeats 42 times>}, mask_was_saved = 0}}, priv = {pad
{0x0, 0x0, 0x0, 0x0}, data = {prev = 0x0, cleanup = 0x0, canceltype = 0}}}
not_first_call = <optimized out>
pagesize_m1 = <optimized out>
sp = <optimized out>
freesize = <optimized out>
__PRETTY_FUNCTION__ = "start_thread"
#24 0x00003fffa3de60fc in .__clone () at
../sysdeps/unix/sysv/linux/powerpc/powerpc64/clone.S:96
No locals.
Thread 6 (LWP 1735):
#0 0x00003fffa3ea3ccc in __pthread_cond_timedwait (cond=0x10059a98,
mutex=0x10059a70, abstime=0x3fffa141f670) at pthread_cond_timedwait.c:198
r4 = 393
r7 = 0
arg5 = 0
arg2 = <optimized out>
r5 = 2
r8 = 4294967295
arg6 = 4294967295
arg3 = 2
r0 = 221
r3 = 516
r6 = 70367154665072
arg4 = 70367154665072
arg1 = 268802716
__err = <optimized out>
__ret = <optimized out>
futex_val = 2
buffer = {__routine = @0x3fffa3ec0b50: 0x3fffa3ea3400
<__condvar_cleanup>, __arg = 0x3fffa141f540, __canceltype = 0, __prev =
0x0}
cbuffer = {oldtype = 0, cond = 0x10059a98, mutex = 0x10059a70,
bc_seq = 0}
result = 0
pshared = 0
pi_flag = 0
err = <optimized out>
val = <optimized out>
---Type <return> to continue, or q <return> to quit---
seq = 0
#1 0x00003fffa3fc0e74 in syncenv_task (proc=0x10053eb0) at syncop.c:607
env = 0x10053eb0
task = 0x0
sleep_till = {tv_sec = 1536845230, tv_nsec = 0}
ret = <optimized out>
#2 0x00003fffa3fc1cdc in syncenv_processor (thdata=0x10053eb0) at
syncop.c:699
env = 0x10053eb0
proc = 0x10053eb0
task = <optimized out>
#3 0x00003fffa3e9bb30 in start_thread (arg=0x3fffa1420160) at
pthread_create.c:462
pd = 0x3fffa1420160
now = <optimized out>
unwind_buf = {cancel_jmp_buf = {{jmp_buf = {2380233324731669406,
70367199403008, 2380233324703897146, 0, 0, 70367146283008, 70367154668064,
8388608, 70367199363104, 268779184,
268779184, 70367199388632, 70367200820192, 3, 0,
70367199388648, 70368094386080, 70368094386136, 4001536, 70367199364120,
70367154665280, -3187653564,
0 <repeats 42 times>}, mask_was_saved = 0}}, priv = {pad
{0x0, 0x0, 0x0, 0x0}, data = {prev = 0x0, cleanup = 0x0, canceltype = 0}}}
not_first_call = <optimized out>
pagesize_m1 = <optimized out>
sp = <optimized out>
freesize = <optimized out>
__PRETTY_FUNCTION__ = "start_thread"
#4 0x00003fffa3de60fc in .__clone () at
../sysdeps/unix/sysv/linux/powerpc/powerpc64/clone.S:96
No locals.
Thread 5 (LWP 1746):
#0 0x00003fffa3ea7e38 in __lll_lock_wait (futex=0x100272a8,
private=<optimized out>) at lowlevellock.c:46
r4 = 128
r7 = 0
r5 = 2
r8 = 1
arg3 = 2
r0 = 221
r3 = 512
r6 = 0
arg4 = 0
arg1 = 268595880
__err = <optimized out>
__ret = <optimized out>
#1 0x00003fffa3e9ef64 in __GI___pthread_mutex_lock (mutex=0x100272a8) at
../nptl/pthread_mutex_lock.c:81
__futex = 0x100272a8
__PRETTY_FUNCTION__ = "__pthread_mutex_lock"
type = <optimized out>
id = <optimized out>
#2 0x00003fffa3f6ce8c in _gf_msg (domain=0x3fffa4009e38 "epoll",
file=0x3fffa4009e28 "event-epoll.c", function=0x3fffa4009db8
<__FUNCTION__.8510> "event_dispatch_epoll_worker",
line=<optimized out>, level=<optimized out>,
errnum=<optimized out>,
trace=<optimized out>, msgid=101190, fmt=0x3fffa4009f48 "Started
thread
with index %d") at logging.c:2058
ret = <optimized out>
---Type <return> to continue, or q <return> to quit---
msgstr = <optimized out>
ap = <optimized out>
this = 0x3fffa402d290 <global_xlator>
ctx = 0x10027010
callstr = '\000' <repeats 4095 times>
passcallstr = 0
log_inited = 0
__PRETTY_FUNCTION__ = "_gf_msg"
#3 0x00003fffa3fe265c in event_dispatch_epoll_worker (data=0x3fff9802ffd0)
at event-epoll.c:631
event = {events = 0, data = {ptr = 0x0, fd = 0, u32 = 0, u64 = 0}}
ret = -1
ev_data = 0x3fff9802ffd0
event_pool = 0x10045bc0
myindex = <optimized out>
timetodie = 0
__FUNCTION__ = "event_dispatch_epoll_worker"
#4 0x00003fffa3e9bb30 in start_thread (arg=0x3fff9f37d160) at
pthread_create.c:462
pd = 0x3fff9f37d160
now = <optimized out>
unwind_buf = {cancel_jmp_buf = {{jmp_buf = {2380233325439331230,
70367199403008, 2380233324703897146, 0, 269006880, 70367112060928,
70367120445984, 8388608, 70367199363104,
70366999543760, 70367137025008, 70367199388632,
70367200825640, 3, 0, 70367199388648, 70367137024656, 70367137024712,
4001536, 70367199364120, 70367120443200, -3183328188,
0 <repeats 42 times>}, mask_was_saved = 0}}, priv = {pad
{0x0, 0x0, 0x0, 0x0}, data = {prev = 0x0, cleanup = 0x0, canceltype = 0}}}
not_first_call = <optimized out>
pagesize_m1 = <optimized out>
sp = <optimized out>
freesize = <optimized out>
__PRETTY_FUNCTION__ = "start_thread"
#5 0x00003fffa3de60fc in .__clone () at
../sysdeps/unix/sysv/linux/powerpc/powerpc64/clone.S:96
No locals.
Thread 4 (LWP 1733):
#0 0x00003fffa3ea9150 in .__nanosleep () at
../sysdeps/unix/syscall-template.S:84
No locals.
#1 0x00003fffa3f93f28 in gf_timer_proc (ctx=0x10027010) at timer.c:205
now = 20528921491
now_ts = {tv_sec = 20, tv_nsec = 528921491}
reg = 0x100533c0
sleepts = {tv_sec = 0, tv_nsec = 2000000}
event = 0x1008d280
old_THIS = <optimized out>
__FUNCTION__ = "gf_timer_proc"
#2 0x00003fffa3e9bb30 in start_thread (arg=0x3fffa2420160) at
pthread_create.c:462
pd = 0x3fffa2420160
now = <optimized out>
unwind_buf = {cancel_jmp_buf = {{jmp_buf = {2380233324681337758,
70367199403008, 2380233324703897146, 0, 0, 70367163060224, 70367171445280,
8388608, 70367199363104, 268595216,
268776384, 70367199388632, 70367200807088, 3, 0,
70367199388648, 70368094385584, 70368094385640, 4001536, 70367199364120,
70367171442496, -3187653564,
---Type <return> to continue, or q <return> to quit---
0 <repeats 42 times>}, mask_was_saved = 0}}, priv = {pad
{0x0, 0x0, 0x0, 0x0}, data = {prev = 0x0, cleanup = 0x0, canceltype = 0}}}
not_first_call = <optimized out>
pagesize_m1 = <optimized out>
sp = <optimized out>
freesize = <optimized out>
__PRETTY_FUNCTION__ = "start_thread"
#3 0x00003fffa3de60fc in .__clone () at
../sysdeps/unix/sysv/linux/powerpc/powerpc64/clone.S:96
No locals.
Thread 3 (LWP 1732):
#0 0x00003fffa3e9d084 in pthread_join (threadid=70367137030496,
thread_return=0x0) at pthread_join.c:90
r4 = 0
r7 = 2
arg2 = 0
r5 = 1743
r8 = 2
arg3 = 1743
r0 = 221
r3 = 512
r6 = 0
arg4 = 0
arg1 = 70367137030704
__err = <optimized out>
__ret = <optimized out>
__tid = 1743
_buffer = {__routine = @0x3fffa3ec0478: 0x3fffa3e9cf70 <cleanup>,
__arg = 0x3fffa034e588, __canceltype = 16383, __prev = 0x0}
oldtype = 0
self = 0x3fffa405e000
result = 0
#1 0x00003fffa3fe2ca0 in event_dispatch_epoll (event_pool=0x10045bc0) at
event-epoll.c:762
i = <optimized out>
t_id = 70367137030496
pollercount = 1
ret = 0
ev_data = <optimized out>
__FUNCTION__ = "event_dispatch_epoll"
#2 0x00003fffa3fab7e4 in event_dispatch (event_pool=<optimized out>) at
event.c:128
ret = -1
__FUNCTION__ = "event_dispatch"
#3 0x0000000010005ea8 in main (argc=<optimized out>, argv=<optimized
out>)
at glusterfsd.c:2380
ctx = 0x10027010
ret = 0
cmdlinestr = "/usr/sbin/glusterfs -s localhost --volfile-id
gluster/glustershd -p /system/glusterd/glustershd/run/glustershd.pid -l
/var/log/glusterfs/glustershd.log -S
/var/run/gluster/868ec0f3795bfb5abd176dadece9"...
cmd = 0x10027010
__FUNCTION__ = <error reading variable __FUNCTION__ (Cannot access
memory at address 0x10010f00)>
---Type <return> to continue, or q <return> to quit---
#4 0x00003fffa3cfe318 in generic_start_main (main=0x10025718 <main>,
argc=<optimized out>, argv=0x3fffd9450fd8, auxvec=0x3fffd9451068,
init=<optimized out>, rtld_fini=<optimized out>,
stack_end=<optimized out>, fini=<optimized out>) at
../csu/libc-start.c:289
self = 0x3fffa405e000
result = <optimized out>
unwind_buf = {cancel_jmp_buf = {{jmp_buf = {2380233326476224334,
70367199177216, 2380233324705844766, 0 <repeats 13 times>, 70367201122640,
0, 70367201122648, 0, 268589712,
-3724539774, 0 <repeats 42 times>}, mask_was_saved = 0}},
priv = {pad = {0x0, 0x0, 0x3fffd9450ed0, 0x0}, data = {prev = 0x0, cleanup
= 0x0, canceltype = 16383}}}
not_first_call = <optimized out>
#5 0x00003fffa3cfe54c in __libc_start_main (argc=<optimized out>,
argv=<optimized out>, ev=<optimized out>, auxvec=<optimized
out>,
rtld_fini=<optimized out>, stinfo=<optimized out>,
stack_on_entry=<optimized out>) at
../sysdeps/unix/sysv/linux/powerpc/libc-start.c:93
No locals.
#6 0x0000000000000000 in ?? ()
No symbol table info available.
Thread 2 (LWP 1736):
#0 0x00003fffa3ea3ccc in __pthread_cond_timedwait (cond=0x10059a98,
mutex=0x10059a70, abstime=0x3fffa0c1f670) at pthread_cond_timedwait.c:198
r4 = 393
r7 = 0
arg5 = 0
arg2 = <optimized out>
r5 = 2
r8 = 4294967295
arg6 = 4294967295
arg3 = 2
r0 = 221
r3 = 516
r6 = 70367146276464
arg4 = 70367146276464
arg1 = 268802716
__err = <optimized out>
__ret = <optimized out>
futex_val = 2
buffer = {__routine = @0x3fffa3ec0b50: 0x3fffa3ea3400
<__condvar_cleanup>, __arg = 0x3fffa0c1f540, __canceltype = 0, __prev =
0x0}
cbuffer = {oldtype = 0, cond = 0x10059a98, mutex = 0x10059a70,
bc_seq = 0}
result = 0
pshared = 0
pi_flag = 0
err = <optimized out>
val = <optimized out>
seq = 0
#1 0x00003fffa3fc0e74 in syncenv_task (proc=0x10054468) at syncop.c:607
env = 0x10053eb0
task = 0x0
sleep_till = {tv_sec = 1536845230, tv_nsec = 0}
ret = <optimized out>
#2 0x00003fffa3fc1cdc in syncenv_processor (thdata=0x10054468) at
syncop.c:699
env = 0x10053eb0
---Type <return> to continue, or q <return> to quit---
proc = 0x10054468
task = <optimized out>
#3 0x00003fffa3e9bb30 in start_thread (arg=0x3fffa0c20160) at
pthread_create.c:462
pd = 0x3fffa0c20160
now = <optimized out>
unwind_buf = {cancel_jmp_buf = {{jmp_buf = {2380233324723280798,
70367199403008, 2380233324703897146, 0, 0, 70367137894400, 70367146279456,
8388608, 70367199363104, 268780648,
268780648, 70367199388632, 70367200820192, 3, 0,
70367199388648, 70368094386080, 70368094386136, 4001536, 70367199364120,
70367146276672, -3187653564,
0 <repeats 42 times>}, mask_was_saved = 0}}, priv = {pad
{0x0, 0x0, 0x0, 0x0}, data = {prev = 0x0, cleanup = 0x0, canceltype = 0}}}
not_first_call = <optimized out>
pagesize_m1 = <optimized out>
sp = <optimized out>
freesize = <optimized out>
__PRETTY_FUNCTION__ = "start_thread"
#4 0x00003fffa3de60fc in .__clone () at
../sysdeps/unix/sysv/linux/powerpc/powerpc64/clone.S:96
No locals.
Thread 1 (LWP 1734):
#0 0x00003fffa3d643b0 in _IO_unbuffer_all () at genops.c:960
__result = 0
__self = 0x3fffa1c20160
cnt = 1
fp = 0x3fff98001f00
#1 _IO_cleanup () at genops.c:1020
result = 0
#2 0x00003fffa3d1ed00 in __run_exit_handlers (status=<optimized out>,
listp=<optimized out>, run_list_atexit=run_list_atexit at entry=true) at
exit.c:95
ptr = 0x3fffa3e75000
<__elf_set___libc_atexit_element__IO_cleanup__>
#3 0x00003fffa3d1ee1c in __GI_exit (status=<optimized out>) at exit.c:104
No locals.
#4 0x000000001000984c in cleanup_and_exit (signum=<optimized out>) at
glusterfsd.c:1295
ctx = <optimized out>
trav = <optimized out>
__FUNCTION__ = <error reading variable __FUNCTION__ (Cannot access
memory at address 0x10010e38)>
#5 0x0000000010009a64 in glusterfs_sigwaiter (arg=<optimized out>) at
glusterfsd.c:2016
set = {__val = {18947, 0 <repeats 15 times>}}
ret = <optimized out>
sig = 15
#6 0x00003fffa3e9bb30 in start_thread (arg=0x3fffa1c20160) at
pthread_create.c:462
pd = 0x3fffa1c20160
now = <optimized out>
unwind_buf = {cancel_jmp_buf = {{jmp_buf = {2380233324740058014,
70367199403008, 2380233324703897146, 0, 0, 70367154671616, 70367163056672,
8388608, 70367199363104,
70368094386576, 268595776, 70367199388632, 268588656, 3, 0,
70367199388648, 70368094386224, 70368094386280, 4001536, 70367199364120,
70367163053888, -3187653564,
0 <repeats 42 times>}, mask_was_saved = 0}}, priv = {pad
{0x0, 0x0, 0x0, 0x0}, data = {prev = 0x0, cleanup = 0x0, canceltype = 0}}}
not_first_call = <optimized out>
pagesize_m1 = <optimized out>
sp = <optimized out>
freesize = <optimized out>
---Type <return> to continue, or q <return> to quit---
__PRETTY_FUNCTION__ = "start_thread"
#7 0x00003fffa3de60fc in .__clone () at
../sysdeps/unix/sysv/linux/powerpc/powerpc64/clone.S:96
No locals.
(gdb)
(gdb)
On Fri, Sep 21, 2018 at 3:32 PM ABHISHEK PALIWAL <abhishpaliwal at
gmail.com>
wrote:
> Hi Sanju,
>
> Here is the output of 't a a bt'
>
> (gdb) t a a bt
>
> Thread 7 (LWP 444):
> #0 0x00003fff7a4d4ccc in __pthread_cond_timedwait (cond=0x10059a98,
> mutex=0x10059a70, abstime=0x3fff77a50670) at pthread_cond_timedwait.c:198
> #1 0x00003fff7a5f1e74 in syncenv_task (proc=0x10053eb0) at syncop.c:607
> #2 0x00003fff7a5f2cdc in syncenv_processor (thdata=0x10053eb0) at
> syncop.c:699
> #3 0x00003fff7a4ccb30 in start_thread (arg=0x3fff77a51160) at
> pthread_create.c:462
> #4 0x00003fff7a4170fc in .__clone () at
> ../sysdeps/unix/sysv/linux/powerpc/powerpc64/clone.S:96
>
> Thread 6 (LWP 447):
> #0 0x00003fff7a4d8e38 in __lll_lock_wait (futex=0x100272a8,
> private=<optimized out>) at lowlevellock.c:46
> #1 0x00003fff7a4cff64 in __GI___pthread_mutex_lock (mutex=0x100272a8) at
> ../nptl/pthread_mutex_lock.c:81
> #2 0x00003fff7a59de8c in _gf_msg (domain=0x3fff70006c90
> "c_glusterfs-client-1", file=0x3fff76165de0 "client.c",
> function=0x3fff76165cd8 <__FUNCTION__.18849> "notify",
line=<optimized
> out>, level=<optimized out>, errnum=<optimized out>,
trace=<optimized out>,
> msgid=114020,
> fmt=0x3fff76166350 "parent translators are ready, attempting
connect
> on transport") at logging.c:2058
> #3 0x00003fff761394ac in notify (this=0x3fff700061f0, event=<optimized
> out>, data=0x3fff70008c50) at client.c:2116
> #4 0x00003fff7a599ca0 in xlator_notify (xl=0x3fff700061f0,
> event=<optimized out>, data=<optimized out>) at xlator.c:491
> #5 0x00003fff7a5b8700 in default_notify (this=0x3fff70008c50,
> event=<optimized out>, data=<optimized out>) at defaults.c:2302
> #6 0x00003fff760fa22c in afr_notify (this=0x3fff70008c50, event=1,
> data=0x3fff7000a4c0, data2=<optimized out>) at afr-common.c:3967
> #7 0x00003fff76105994 in notify (this=<optimized out>,
event=<optimized
> out>, data=<optimized out>) at afr.c:38
> #8 0x00003fff7a599ca0 in xlator_notify (xl=0x3fff70008c50,
> event=<optimized out>, data=<optimized out>) at xlator.c:491
> #9 0x00003fff7a5b8700 in default_notify (this=0x3fff7000a4c0,
> event=<optimized out>, data=<optimized out>) at defaults.c:2302
> #10 0x00003fff7609c1e4 in notify (this=<optimized out>,
event=<optimized
> out>, data=<optimized out>) at io-stats.c:3064
> #11 0x00003fff7a599ca0 in xlator_notify (xl=0x3fff7000a4c0,
> event=<optimized out>, data=<optimized out>) at xlator.c:491
> #12 0x00003fff7a5ee560 in glusterfs_graph_parent_up (graph=<optimized
> out>) at graph.c:440
> #13 0x00003fff7a5eeb90 in glusterfs_graph_activate (graph=0x3fff70000af0,
> ctx=0x10027010) at graph.c:688
> #14 0x000000001000a49c in glusterfs_process_volfp (ctx=0x10027010,
> fp=0x3fff70001cd0) at glusterfsd.c:2221
> #15 0x000000001000fd08 in mgmt_getspec_cbk (req=<optimized out>,
> iov=<optimized out>, count=<optimized out>,
myframe=0x3fff7921b06c) at
> glusterfsd-mgmt.c:1561
> #16 0x00003fff7a552ec4 in rpc_clnt_handle_reply (clnt=0x10089020,
> pollin=0x3fff70001760) at rpc-clnt.c:775
> #17 0x00003fff7a5533d0 in rpc_clnt_notify (trans=<optimized out>,
> mydata=0x10089050, event=<optimized out>, data=<optimized out>)
at
> rpc-clnt.c:933
> #18 0x00003fff7a54e4fc in rpc_transport_notify (this=<optimized out>,
> event=<optimized out>, data=<optimized out>) at
rpc-transport.c:546
> #19 0x00003fff76a32d44 in socket_event_poll_in (this=this at
entry=0x1008ab80)
> at socket.c:2236
> #20 0x00003fff76a3589c in socket_event_handler (fd=<optimized out>,
> idx=<optimized out>, data=0x1008ab80, poll_in=<optimized out>,
> poll_out=<optimized out>,
> ---Type <return> to continue, or q <return> to quit---
> poll_err=<optimized out>) at socket.c:2349
> #21 0x00003fff7a613874 in event_dispatch_epoll_handler
> (event=0x3fff7697e6a0, event_pool=0x10045bc0) at event-epoll.c:575
> #22 event_dispatch_epoll_worker (data=0x1008bd50) at event-epoll.c:678
> #23 0x00003fff7a4ccb30 in start_thread (arg=0x3fff7697f160) at
> pthread_create.c:462
> #24 0x00003fff7a4170fc in .__clone () at
> ../sysdeps/unix/sysv/linux/powerpc/powerpc64/clone.S:96
>
> Thread 5 (LWP 453):
> #0 0x00003fff7a4d8e38 in __lll_lock_wait (futex=0x100272a8,
> private=<optimized out>) at lowlevellock.c:46
> #1 0x00003fff7a4cff64 in __GI___pthread_mutex_lock (mutex=0x100272a8) at
> ../nptl/pthread_mutex_lock.c:81
> #2 0x00003fff7a59de8c in _gf_msg (domain=0x3fff7a63ae38 "epoll",
> file=0x3fff7a63ae28 "event-epoll.c",
> function=0x3fff7a63adb8 <__FUNCTION__.8510>
> "event_dispatch_epoll_worker", line=<optimized out>,
level=<optimized out>,
> errnum=<optimized out>,
> trace=<optimized out>, msgid=101190, fmt=0x3fff7a63af48
"Started
> thread with index %d") at logging.c:2058
> #3 0x00003fff7a61365c in event_dispatch_epoll_worker
> (data=0x3fff7002ffd0) at event-epoll.c:631
> #4 0x00003fff7a4ccb30 in start_thread (arg=0x3fff759ae160) at
> pthread_create.c:462
> #5 0x00003fff7a4170fc in .__clone () at
> ../sysdeps/unix/sysv/linux/powerpc/powerpc64/clone.S:96
>
> Thread 4 (LWP 441):
> #0 0x00003fff7a4ce084 in pthread_join (threadid=70366438879584,
> thread_return=0x0) at pthread_join.c:90
> #1 0x00003fff7a613ca0 in event_dispatch_epoll (event_pool=0x10045bc0) at
> event-epoll.c:762
> #2 0x00003fff7a5dc7e4 in event_dispatch (event_pool=<optimized out>)
at
> event.c:128
> #3 0x0000000010005ea8 in main (argc=<optimized out>,
argv=<optimized
> out>) at glusterfsd.c:2380
> #4 0x00003fff7a32f318 in generic_start_main (main=0x10025718 <main>,
> argc=<optimized out>, argv=0x3fffd839d7a8, auxvec=0x3fffd839d838,
> init=<optimized out>,
> rtld_fini=<optimized out>, stack_end=<optimized out>,
fini=<optimized
> out>) at ../csu/libc-start.c:289
> #5 0x00003fff7a32f54c in __libc_start_main (argc=<optimized out>,
> argv=<optimized out>, ev=<optimized out>, auxvec=<optimized
out>,
> rtld_fini=<optimized out>,
> stinfo=<optimized out>, stack_on_entry=<optimized out>) at
> ../sysdeps/unix/sysv/linux/powerpc/libc-start.c:93
> #6 0x0000000000000000 in ?? ()
>
> Thread 3 (LWP 442):
> #0 0x00003fff7a4da150 in .__nanosleep () at
> ../sysdeps/unix/syscall-template.S:84
> #1 0x00003fff7a5c4f28 in gf_timer_proc (ctx=0x10027010) at timer.c:205
> #2 0x00003fff7a4ccb30 in start_thread (arg=0x3fff78a51160) at
> pthread_create.c:462
> #3 0x00003fff7a4170fc in .__clone () at
> ../sysdeps/unix/sysv/linux/powerpc/powerpc64/clone.S:96
> ---Type <return> to continue, or q <return> to quit---
>
> Thread 2 (LWP 445):
> #0 0x00003fff7a4d4ccc in __pthread_cond_timedwait (cond=0x10059a98,
> mutex=0x10059a70, abstime=0x3fff77250670) at pthread_cond_timedwait.c:198
> #1 0x00003fff7a5f1e74 in syncenv_task (proc=0x10054468) at syncop.c:607
> #2 0x00003fff7a5f2cdc in syncenv_processor (thdata=0x10054468) at
> syncop.c:699
> #3 0x00003fff7a4ccb30 in start_thread (arg=0x3fff77251160) at
> pthread_create.c:462
> #4 0x00003fff7a4170fc in .__clone () at
> ../sysdeps/unix/sysv/linux/powerpc/powerpc64/clone.S:96
>
> Thread 1 (LWP 443):
> #0 0x00003fff7a3953b0 in _IO_unbuffer_all () at genops.c:960
> #1 _IO_cleanup () at genops.c:1020
> #2 0x00003fff7a34fd00 in __run_exit_handlers (status=<optimized
out>,
> listp=<optimized out>, run_list_atexit=run_list_atexit at entry=true)
at
> exit.c:95
> #3 0x00003fff7a34fe1c in __GI_exit (status=<optimized out>) at
exit.c:104
>
> #4 0x000000001000984c in cleanup_and_exit (signum=<optimized out>)
at
> glusterfsd.c:1295
> #5 0x0000000010009a64 in glusterfs_sigwaiter (arg=<optimized out>)
at
> glusterfsd.c:2016
> #6 0x00003fff7a4ccb30 in start_thread (arg=0x3fff78251160) at
> pthread_create.c:462
> #7 0x00003fff7a4170fc in .__clone () at
> ../sysdeps/unix/sysv/linux/powerpc/powerpc64/clone.S:96
> (gdb)
> (gdb)
>
> On Fri, Sep 21, 2018 at 3:06 PM Sanju Rakonde <srakonde at
redhat.com> wrote:
>
>> Hi Abhishek,
>>
>> Can you please share the output of "t a a bt" with us?
>>
>> Thanks,
>> Sanju
>>
>> On Fri, Sep 21, 2018 at 2:55 PM, ABHISHEK PALIWAL <
>> abhishpaliwal at gmail.com> wrote:
>>
>>>
>>> We have seen a SIGSEGV crash on glusterfs process on kernel restart
at
>>> start up.
>>>
>>> (gdb) bt
>>> #0 0x00003fffad4463b0 in _IO_unbuffer_all () at genops.c:960
>>> #1 _IO_cleanup () at genops.c:1020
>>> #2 0x00003fffad400d00 in __run_exit_handlers (status=<optimized
out>,
>>> listp=<optimized out>, run_list_atexit=run_list_atexit at
entry=true) at
>>> exit.c:95
>>> #3 0x00003fffad400e1c in __GI_exit (status=<optimized out>)
at
>>> exit.c:104
>>> #4 0x000000001000984c in cleanup_and_exit (signum=<optimized
out>) at
>>> glusterfsd.c:1295
>>> #5 0x0000000010009a64 in *glusterfs_sigwaiter *(arg=<optimized
out>)
>>> at glusterfsd.c:2016
>>> #6 0x00003fffad57db30 in start_thread (arg=0x3fffab302160) at
>>> pthread_create.c:462
>>> #7 0x00003fffad4c7cdc in .__clone () at
>>> ../sysdeps/unix/sysv/linux/powerpc/powerpc64/clone.S:96
>>>
>>> (gdb) bt full
>>> #0 0x00003fffad4463b0 in _IO_unbuffer_all () at genops.c:960
>>> __result = 0
>>> __self = 0x3fffab302160
>>> cnt = 1
>>> fp = 0x3fffa4001f00
>>> #1 _IO_cleanup () at genops.c:1020
>>> result = 0
>>> #2 0x00003fffad400d00 in __run_exit_handlers (status=<optimized
out>,
>>> listp=<optimized out>, run_list_atexit=run_list_atexit at
entry=true) at
>>> exit.c:95
>>> ptr = 0x3fffad557000
<__elf_set___libc_atexit_element__IO_cleanup__>
>>> #3 0x00003fffad400e1c in __GI_exit (status=<optimized out>)
at
>>> exit.c:104
>>> No locals.
>>> #4 0x000000001000984c in cleanup_and_exit (signum=<optimized
out>) at
>>> glusterfsd.c:1295
>>> ctx = <optimized out>
>>> trav = <optimized out>
>>> __FUNCTION__ = <error reading variable __FUNCTION__ (Cannot
access
>>> memory at address 0x10010e38)>
>>> #5 0x0000000010009a64 in glusterfs_sigwaiter (arg=<optimized
out>) at
>>> glusterfsd.c:2016
>>> set = {__val = {18947, 0 <repeats 15 times>}}
>>> ret = <optimized out>
>>> sig = 15
>>> #6 0x00003fffad57db30 in start_thread (arg=0x3fffab302160) at
>>> pthread_create.c:462
>>> pd = 0x3fffab302160
>>> now = <optimized out>
>>> unwind_buf = {cancel_jmp_buf = {{jmp_buf =
{5451414826039278896,
>>> 70367357615104, 5451414826003312788, 0, 0, 70367312883712,
70367321268768,
>>> 8388608,
>>> 70367357575200, 70367913735952, 268595776,
70367357600728,
>>> 268588656, 3, 0, 70367357600744, 70367913735600, 70367913735656,
4001536,
>>> 70367357576216, 70367321265984, -3187653564, 0
<repeats 42
>>> times>}, mask_was_saved = 0}}, priv = {pad = {0x0, 0x0, 0x0,
0x0}, data >>> {prev = 0x0,
>>> cleanup = 0x0, canceltype = 0}}}
>>> not_first_call = <optimized out>
>>> pagesize_m1 = <optimized out>
>>> sp = <optimized out>
>>> freesize = <optimized out>
>>> __PRETTY_FUNCTION__ = "start_thread"
>>> ---Type <return> to continue, or q <return> to quit---
>>> #7 0x00003fffad4c7cdc in .__clone () at
>>> ../sysdeps/unix/sysv/linux/powerpc/powerpc64/clone.S:96
>>> No locals
>>>
>>> *Can you please help us in finding the cause for SIGSEGV. ?*
>>> *Also please share your understanding on this issue.*
>>> --
>>> Regards
>>> Abhishek Paliwal
>>>
>>> _______________________________________________
>>> Gluster-devel mailing list
>>> Gluster-devel at gluster.org
>>> https://lists.gluster.org/mailman/listinfo/gluster-devel
>>>
>>
>>
>>
>> --
>> Thanks,
>> Sanju
>>
>
>
> --
>
>
>
>
> Regards
> Abhishek Paliwal
>
--
Regards
Abhishek Paliwal
-------------- next part --------------
An HTML attachment was scrubbed...
URL:
<http://lists.gluster.org/pipermail/gluster-users/attachments/20180921/c6518666/attachment.html>
ABHISHEK PALIWAL
2018-Sep-24 08:05 UTC
[Gluster-users] [Gluster-devel] Crash in glusterfs!!!
Hi Sanju, Do you have any update on this? Regards, Abhishek On Fri, Sep 21, 2018 at 4:07 PM ABHISHEK PALIWAL <abhishpaliwal at gmail.com> wrote:> Hi Sanju, > > Output of 't a a bt full' > > (gdb) t a a bt full > > > > Thread 7 (LWP 1743): > > #0 0x00003fffa3ea7e88 in __lll_lock_wait (futex=0x0, private=0) at > lowlevellock.c:43 > > r4 = 128 > > r7 = 0 > > arg2 = 128 > > r5 = 2 > > r8 = 1 > > r0 = 221 > > r3 = 0 > > r6 = 0 > > arg1 = 0 > > __err = 221 > > __ret = 0 > > #1 0x00003fffa3e9ef64 in __GI___pthread_mutex_lock (mutex=0x100272a8) at > ../nptl/pthread_mutex_lock.c:81 > > __futex = 0x100272a8 > > __PRETTY_FUNCTION__ = "__pthread_mutex_lock" > > type = <optimized out> > > id = <optimized out> > > #2 0x00003fffa3f6ce8c in _gf_msg (domain=0x3fff98006c90 > "c_glusterfs-client-0", file=0x3fff9fb34de0 "client.c", > function=0x3fff9fb34cd8 <__FUNCTION__.18849> "notify", > > line=<optimized out>, level=<optimized out>, errnum=<optimized out>, > trace=<optimized out>, msgid=114020, > > fmt=0x3fff9fb35350 "parent translators are ready, attempting connect > on transport") at logging.c:2058 > > ret = <optimized out> > > msgstr = <optimized out> > > ap = <optimized out> > > this = 0x3fff980061f0 > > ctx = 0x10027010 > > callstr = '\000' <repeats 4095 times> > > passcallstr = 0 > > log_inited = 0 > > __PRETTY_FUNCTION__ = "_gf_msg" > > #3 0x00003fff9fb084ac in notify (this=0x3fff980061f0, event=<optimized > out>, data=0x3fff98008c50) at client.c:2116 > > conf = 0x3fff98056dd0 > > __FUNCTION__ = "notify" > > #4 0x00003fffa3f68ca0 in xlator_notify (xl=0x3fff980061f0, > event=<optimized out>, data=<optimized out>) at xlator.c:491 > > old_THIS = 0x3fff98008c50 > > ret = 0 > > #5 0x00003fffa3f87700 in default_notify (this=0x3fff98008c50, > event=<optimized out>, data=<optimized out>) at defaults.c:2302 > > list = 0x3fff9800a340 > > #6 0x00003fff9fac922c in afr_notify (this=0x3fff98008c50, event=1, > data=0x3fff9800a4c0, data2=<optimized out>) at afr-common.c:3967 > > priv = 0x3fff98010050 > > i = <optimized out> > > up_children = <optimized out> > > down_children = <optimized out> > > propagate = 1 > > had_heard_from_all = <optimized out> > > ---Type <return> to continue, or q <return> to quit--- > > have_heard_from_all = 0 > > idx = <optimized out> > > ret = 0 > > call_psh = <optimized out> > > input = 0x0 > > output = 0x0 > > had_quorum = <optimized out> > > has_quorum = <optimized out> > > __FUNCTION__ = "afr_notify" > > #7 0x00003fff9fad4994 in notify (this=<optimized out>, event=<optimized > out>, data=<optimized out>) at afr.c:38 > > ret = -1 > > ap = 0x3fffa034cc58 "" > > data2 = <optimized out> > > #8 0x00003fffa3f68ca0 in xlator_notify (xl=0x3fff98008c50, > event=<optimized out>, data=<optimized out>) at xlator.c:491 > > old_THIS = 0x3fff9800a4c0 > > ret = 0 > > #9 0x00003fffa3f87700 in default_notify (this=0x3fff9800a4c0, > event=<optimized out>, data=<optimized out>) at defaults.c:2302 > > list = 0x3fff9800b710 > > #10 0x00003fff9fa6b1e4 in notify (this=<optimized out>, event=<optimized > out>, data=<optimized out>) at io-stats.c:3064 > > ret = 0 > > args = {type = IOS_DUMP_TYPE_NONE, u = {logfp = 0x0, dict = 0x0}} > > op = 0 > > list_cnt = 0 > > throughput = 0 > > time = 0 > > is_peek = _gf_false > > ap = 0x3fffa034ce68 "" > > __FUNCTION__ = "notify" > > #11 0x00003fffa3f68ca0 in xlator_notify (xl=0x3fff9800a4c0, > event=<optimized out>, data=<optimized out>) at xlator.c:491 > > old_THIS = 0x3fffa402d290 <global_xlator> > > ret = 0 > > #12 0x00003fffa3fbd560 in glusterfs_graph_parent_up (graph=<optimized > out>) at graph.c:440 > > trav = 0x3fff9800a4c0 > > ret = <optimized out> > > #13 0x00003fffa3fbdb90 in glusterfs_graph_activate (graph=0x3fff98000af0, > ctx=0x10027010) at graph.c:688 > > ret = <optimized out> > > __FUNCTION__ = "glusterfs_graph_activate" > > #14 0x000000001000a49c in glusterfs_process_volfp (ctx=0x10027010, > fp=0x3fff98001cd0) at glusterfsd.c:2221 > > graph = 0x3fff98000af0 > > ret = <optimized out> > > trav = <optimized out> > > __FUNCTION__ = <error reading variable __FUNCTION__ (Cannot access > memory at address 0x10010ec0)> > > #15 0x000000001000fd08 in mgmt_getspec_cbk (req=<optimized out>, > iov=<optimized out>, count=<optimized out>, myframe=0x3fffa2bea06c) at > glusterfsd-mgmt.c:1561 > > rsp = {op_ret = 1059, op_errno = 0, > > spec = 0x3fff980018a0 "volume c_glusterfs-client-0\n type > protocol/client\n option password > 5fd8d83d-99f3-4630-97c6-965d7a8ead62\n option username > e65687aa-e135-445e-8778-48bb8fb19640\n option transport-type tcp\n > "..., xdata = {xdata_len = 0, xdata_val = 0x0}} > > ---Type <return> to continue, or q <return> to quit--- > > frame = 0x3fffa2bea06c > > ctx = 0x10027010 > > ret = <optimized out> > > size = 1059 > > tmpfp = 0x3fff98001cd0 > > volfilebuf = 0x0 > > __FUNCTION__ = <error reading variable __FUNCTION__ (Cannot access > memory at address 0x10013570)> > > #16 0x00003fffa3f21ec4 in rpc_clnt_handle_reply (clnt=0x10089020, > pollin=0x3fff98001760) at rpc-clnt.c:775 > > conn = 0x10089050 > > saved_frame = <optimized out> > > ret = <optimized out> > > req = 0x1008931c > > xid = 1 > > __FUNCTION__ = "rpc_clnt_handle_reply" > > #17 0x00003fffa3f223d0 in rpc_clnt_notify (trans=<optimized out>, > mydata=0x10089050, event=<optimized out>, data=<optimized out>) at > rpc-clnt.c:933 > > conn = 0x10089050 > > clnt = <optimized out> > > ret = -1 > > req_info = 0x0 > > pollin = <optimized out> > > clnt_mydata = 0x0 > > old_THIS = 0x3fffa402d290 <global_xlator> > > __FUNCTION__ = "rpc_clnt_notify" > > #18 0x00003fffa3f1d4fc in rpc_transport_notify (this=<optimized out>, > event=<optimized out>, data=<optimized out>) at rpc-transport.c:546 > > ret = -1 > > __FUNCTION__ = "rpc_transport_notify" > > #19 0x00003fffa0401d44 in socket_event_poll_in (this=this at entry=0x1008ab80) > at socket.c:2236 > > ret = <optimized out> > > pollin = 0x3fff98001760 > > priv = 0x1008b820 > > #20 0x00003fffa040489c in socket_event_handler (fd=<optimized out>, > idx=<optimized out>, data=0x1008ab80, poll_in=<optimized out>, > poll_out=<optimized out>, poll_err=<optimized out>) > > at socket.c:2349 > > this = 0x1008ab80 > > priv = 0x1008b820 > > ret = <optimized out> > > __FUNCTION__ = "socket_event_handler" > > #21 0x00003fffa3fe2874 in event_dispatch_epoll_handler > (event=0x3fffa034d6a0, event_pool=0x10045bc0) at event-epoll.c:575 > > handler = @0x3fffa041f620: 0x3fffa04046f0 <socket_event_handler> > > gen = 1 > > slot = 0x1007cd80 > > data = <optimized out> > > ret = -1 > > fd = 9 > > ev_data = 0x3fffa034d6a8 > > idx = 1 > > #22 event_dispatch_epoll_worker (data=0x1008bd50) at event-epoll.c:678 > > ---Type <return> to continue, or q <return> to quit--- > > event = {events = 1, data = {ptr = 0x100000001, fd = 1, u32 = 1, > u64 = 4294967297}} > > ret = <optimized out> > > ev_data = 0x1008bd50 > > event_pool = 0x10045bc0 > > myindex = <optimized out> > > timetodie = 0 > > __FUNCTION__ = "event_dispatch_epoll_worker" > > #23 0x00003fffa3e9bb30 in start_thread (arg=0x3fffa034e160) at > pthread_create.c:462 > > pd = 0x3fffa034e160 > > now = <optimized out> > > unwind_buf = {cancel_jmp_buf = {{jmp_buf = {2380233324717718430, > 70367199403008, 2380233324703897146, 0, 0, 70367128645632, 70367137030688, > 8388608, 70367199363104, 269008208, > > 70368094386592, 70367199388632, 70367200825640, 3, 0, > 70367199388648, 70368094386240, 70368094386296, 4001536, 70367199364120, > 70367137027904, -3187653596, > > 0 <repeats 42 times>}, mask_was_saved = 0}}, priv = {pad > {0x0, 0x0, 0x0, 0x0}, data = {prev = 0x0, cleanup = 0x0, canceltype = 0}}} > > not_first_call = <optimized out> > > pagesize_m1 = <optimized out> > > sp = <optimized out> > > freesize = <optimized out> > > __PRETTY_FUNCTION__ = "start_thread" > > #24 0x00003fffa3de60fc in .__clone () at > ../sysdeps/unix/sysv/linux/powerpc/powerpc64/clone.S:96 > > No locals. > > > > Thread 6 (LWP 1735): > > #0 0x00003fffa3ea3ccc in __pthread_cond_timedwait (cond=0x10059a98, > mutex=0x10059a70, abstime=0x3fffa141f670) at pthread_cond_timedwait.c:198 > > r4 = 393 > > r7 = 0 > > arg5 = 0 > > arg2 = <optimized out> > > r5 = 2 > > r8 = 4294967295 > > arg6 = 4294967295 > > arg3 = 2 > > r0 = 221 > > r3 = 516 > > r6 = 70367154665072 > > arg4 = 70367154665072 > > arg1 = 268802716 > > __err = <optimized out> > > __ret = <optimized out> > > futex_val = 2 > > buffer = {__routine = @0x3fffa3ec0b50: 0x3fffa3ea3400 > <__condvar_cleanup>, __arg = 0x3fffa141f540, __canceltype = 0, __prev = 0x0} > > cbuffer = {oldtype = 0, cond = 0x10059a98, mutex = 0x10059a70, > bc_seq = 0} > > result = 0 > > pshared = 0 > > pi_flag = 0 > > err = <optimized out> > > val = <optimized out> > > ---Type <return> to continue, or q <return> to quit--- > > seq = 0 > > #1 0x00003fffa3fc0e74 in syncenv_task (proc=0x10053eb0) at syncop.c:607 > > env = 0x10053eb0 > > task = 0x0 > > sleep_till = {tv_sec = 1536845230, tv_nsec = 0} > > ret = <optimized out> > > #2 0x00003fffa3fc1cdc in syncenv_processor (thdata=0x10053eb0) at > syncop.c:699 > > env = 0x10053eb0 > > proc = 0x10053eb0 > > task = <optimized out> > > #3 0x00003fffa3e9bb30 in start_thread (arg=0x3fffa1420160) at > pthread_create.c:462 > > pd = 0x3fffa1420160 > > now = <optimized out> > > unwind_buf = {cancel_jmp_buf = {{jmp_buf = {2380233324731669406, > 70367199403008, 2380233324703897146, 0, 0, 70367146283008, 70367154668064, > 8388608, 70367199363104, 268779184, > > 268779184, 70367199388632, 70367200820192, 3, 0, > 70367199388648, 70368094386080, 70368094386136, 4001536, 70367199364120, > 70367154665280, -3187653564, > > 0 <repeats 42 times>}, mask_was_saved = 0}}, priv = {pad > {0x0, 0x0, 0x0, 0x0}, data = {prev = 0x0, cleanup = 0x0, canceltype = 0}}} > > not_first_call = <optimized out> > > pagesize_m1 = <optimized out> > > sp = <optimized out> > > freesize = <optimized out> > > __PRETTY_FUNCTION__ = "start_thread" > > #4 0x00003fffa3de60fc in .__clone () at > ../sysdeps/unix/sysv/linux/powerpc/powerpc64/clone.S:96 > > No locals. > > > > Thread 5 (LWP 1746): > > #0 0x00003fffa3ea7e38 in __lll_lock_wait (futex=0x100272a8, > private=<optimized out>) at lowlevellock.c:46 > > r4 = 128 > > r7 = 0 > > r5 = 2 > > r8 = 1 > > arg3 = 2 > > r0 = 221 > > r3 = 512 > > r6 = 0 > > arg4 = 0 > > arg1 = 268595880 > > __err = <optimized out> > > __ret = <optimized out> > > #1 0x00003fffa3e9ef64 in __GI___pthread_mutex_lock (mutex=0x100272a8) at > ../nptl/pthread_mutex_lock.c:81 > > __futex = 0x100272a8 > > __PRETTY_FUNCTION__ = "__pthread_mutex_lock" > > type = <optimized out> > > id = <optimized out> > > #2 0x00003fffa3f6ce8c in _gf_msg (domain=0x3fffa4009e38 "epoll", > file=0x3fffa4009e28 "event-epoll.c", function=0x3fffa4009db8 > <__FUNCTION__.8510> "event_dispatch_epoll_worker", > > line=<optimized out>, level=<optimized out>, errnum=<optimized out>, > trace=<optimized out>, msgid=101190, fmt=0x3fffa4009f48 "Started thread > with index %d") at logging.c:2058 > > ret = <optimized out> > > ---Type <return> to continue, or q <return> to quit--- > > msgstr = <optimized out> > > ap = <optimized out> > > this = 0x3fffa402d290 <global_xlator> > > ctx = 0x10027010 > > callstr = '\000' <repeats 4095 times> > > passcallstr = 0 > > log_inited = 0 > > __PRETTY_FUNCTION__ = "_gf_msg" > > #3 0x00003fffa3fe265c in event_dispatch_epoll_worker > (data=0x3fff9802ffd0) at event-epoll.c:631 > > event = {events = 0, data = {ptr = 0x0, fd = 0, u32 = 0, u64 = 0}} > > ret = -1 > > ev_data = 0x3fff9802ffd0 > > event_pool = 0x10045bc0 > > myindex = <optimized out> > > timetodie = 0 > > __FUNCTION__ = "event_dispatch_epoll_worker" > > #4 0x00003fffa3e9bb30 in start_thread (arg=0x3fff9f37d160) at > pthread_create.c:462 > > pd = 0x3fff9f37d160 > > now = <optimized out> > > unwind_buf = {cancel_jmp_buf = {{jmp_buf = {2380233325439331230, > 70367199403008, 2380233324703897146, 0, 269006880, 70367112060928, > 70367120445984, 8388608, 70367199363104, > > 70366999543760, 70367137025008, 70367199388632, > 70367200825640, 3, 0, 70367199388648, 70367137024656, 70367137024712, > 4001536, 70367199364120, 70367120443200, -3183328188, > > 0 <repeats 42 times>}, mask_was_saved = 0}}, priv = {pad > {0x0, 0x0, 0x0, 0x0}, data = {prev = 0x0, cleanup = 0x0, canceltype = 0}}} > > not_first_call = <optimized out> > > pagesize_m1 = <optimized out> > > sp = <optimized out> > > freesize = <optimized out> > > __PRETTY_FUNCTION__ = "start_thread" > > #5 0x00003fffa3de60fc in .__clone () at > ../sysdeps/unix/sysv/linux/powerpc/powerpc64/clone.S:96 > > No locals. > > > > Thread 4 (LWP 1733): > > #0 0x00003fffa3ea9150 in .__nanosleep () at > ../sysdeps/unix/syscall-template.S:84 > > No locals. > > #1 0x00003fffa3f93f28 in gf_timer_proc (ctx=0x10027010) at timer.c:205 > > now = 20528921491 > > now_ts = {tv_sec = 20, tv_nsec = 528921491} > > reg = 0x100533c0 > > sleepts = {tv_sec = 0, tv_nsec = 2000000} > > event = 0x1008d280 > > old_THIS = <optimized out> > > __FUNCTION__ = "gf_timer_proc" > > #2 0x00003fffa3e9bb30 in start_thread (arg=0x3fffa2420160) at > pthread_create.c:462 > > pd = 0x3fffa2420160 > > now = <optimized out> > > unwind_buf = {cancel_jmp_buf = {{jmp_buf = {2380233324681337758, > 70367199403008, 2380233324703897146, 0, 0, 70367163060224, 70367171445280, > 8388608, 70367199363104, 268595216, > > 268776384, 70367199388632, 70367200807088, 3, 0, > 70367199388648, 70368094385584, 70368094385640, 4001536, 70367199364120, > 70367171442496, -3187653564, > > ---Type <return> to continue, or q <return> to quit--- > > 0 <repeats 42 times>}, mask_was_saved = 0}}, priv = {pad > {0x0, 0x0, 0x0, 0x0}, data = {prev = 0x0, cleanup = 0x0, canceltype = 0}}} > > not_first_call = <optimized out> > > pagesize_m1 = <optimized out> > > sp = <optimized out> > > freesize = <optimized out> > > __PRETTY_FUNCTION__ = "start_thread" > > #3 0x00003fffa3de60fc in .__clone () at > ../sysdeps/unix/sysv/linux/powerpc/powerpc64/clone.S:96 > > No locals. > > > > Thread 3 (LWP 1732): > > #0 0x00003fffa3e9d084 in pthread_join (threadid=70367137030496, > thread_return=0x0) at pthread_join.c:90 > > r4 = 0 > > r7 = 2 > > arg2 = 0 > > r5 = 1743 > > r8 = 2 > > arg3 = 1743 > > r0 = 221 > > r3 = 512 > > r6 = 0 > > arg4 = 0 > > arg1 = 70367137030704 > > __err = <optimized out> > > __ret = <optimized out> > > __tid = 1743 > > _buffer = {__routine = @0x3fffa3ec0478: 0x3fffa3e9cf70 <cleanup>, > __arg = 0x3fffa034e588, __canceltype = 16383, __prev = 0x0} > > oldtype = 0 > > self = 0x3fffa405e000 > > result = 0 > > #1 0x00003fffa3fe2ca0 in event_dispatch_epoll (event_pool=0x10045bc0) at > event-epoll.c:762 > > i = <optimized out> > > t_id = 70367137030496 > > pollercount = 1 > > ret = 0 > > ev_data = <optimized out> > > __FUNCTION__ = "event_dispatch_epoll" > > #2 0x00003fffa3fab7e4 in event_dispatch (event_pool=<optimized out>) at > event.c:128 > > ret = -1 > > __FUNCTION__ = "event_dispatch" > > #3 0x0000000010005ea8 in main (argc=<optimized out>, argv=<optimized > out>) at glusterfsd.c:2380 > > ctx = 0x10027010 > > ret = 0 > > cmdlinestr = "/usr/sbin/glusterfs -s localhost --volfile-id > gluster/glustershd -p /system/glusterd/glustershd/run/glustershd.pid -l > /var/log/glusterfs/glustershd.log -S > /var/run/gluster/868ec0f3795bfb5abd176dadece9"... > > cmd = 0x10027010 > > __FUNCTION__ = <error reading variable __FUNCTION__ (Cannot access > memory at address 0x10010f00)> > > ---Type <return> to continue, or q <return> to quit--- > > #4 0x00003fffa3cfe318 in generic_start_main (main=0x10025718 <main>, > argc=<optimized out>, argv=0x3fffd9450fd8, auxvec=0x3fffd9451068, > init=<optimized out>, rtld_fini=<optimized out>, > > stack_end=<optimized out>, fini=<optimized out>) at > ../csu/libc-start.c:289 > > self = 0x3fffa405e000 > > result = <optimized out> > > unwind_buf = {cancel_jmp_buf = {{jmp_buf = {2380233326476224334, > 70367199177216, 2380233324705844766, 0 <repeats 13 times>, 70367201122640, > 0, 70367201122648, 0, 268589712, > > -3724539774, 0 <repeats 42 times>}, mask_was_saved = 0}}, > priv = {pad = {0x0, 0x0, 0x3fffd9450ed0, 0x0}, data = {prev = 0x0, cleanup > = 0x0, canceltype = 16383}}} > > not_first_call = <optimized out> > > #5 0x00003fffa3cfe54c in __libc_start_main (argc=<optimized out>, > argv=<optimized out>, ev=<optimized out>, auxvec=<optimized out>, > rtld_fini=<optimized out>, stinfo=<optimized out>, > > stack_on_entry=<optimized out>) at > ../sysdeps/unix/sysv/linux/powerpc/libc-start.c:93 > > No locals. > > #6 0x0000000000000000 in ?? () > > No symbol table info available. > > > > Thread 2 (LWP 1736): > > #0 0x00003fffa3ea3ccc in __pthread_cond_timedwait (cond=0x10059a98, > mutex=0x10059a70, abstime=0x3fffa0c1f670) at pthread_cond_timedwait.c:198 > > r4 = 393 > > r7 = 0 > > arg5 = 0 > > arg2 = <optimized out> > > r5 = 2 > > r8 = 4294967295 > > arg6 = 4294967295 > > arg3 = 2 > > r0 = 221 > > r3 = 516 > > r6 = 70367146276464 > > arg4 = 70367146276464 > > arg1 = 268802716 > > __err = <optimized out> > > __ret = <optimized out> > > futex_val = 2 > > buffer = {__routine = @0x3fffa3ec0b50: 0x3fffa3ea3400 > <__condvar_cleanup>, __arg = 0x3fffa0c1f540, __canceltype = 0, __prev = 0x0} > > cbuffer = {oldtype = 0, cond = 0x10059a98, mutex = 0x10059a70, > bc_seq = 0} > > result = 0 > > pshared = 0 > > pi_flag = 0 > > err = <optimized out> > > val = <optimized out> > > seq = 0 > > #1 0x00003fffa3fc0e74 in syncenv_task (proc=0x10054468) at syncop.c:607 > > env = 0x10053eb0 > > task = 0x0 > > sleep_till = {tv_sec = 1536845230, tv_nsec = 0} > > ret = <optimized out> > > #2 0x00003fffa3fc1cdc in syncenv_processor (thdata=0x10054468) at > syncop.c:699 > > env = 0x10053eb0 > > ---Type <return> to continue, or q <return> to quit--- > > proc = 0x10054468 > > task = <optimized out> > > #3 0x00003fffa3e9bb30 in start_thread (arg=0x3fffa0c20160) at > pthread_create.c:462 > > pd = 0x3fffa0c20160 > > now = <optimized out> > > unwind_buf = {cancel_jmp_buf = {{jmp_buf = {2380233324723280798, > 70367199403008, 2380233324703897146, 0, 0, 70367137894400, 70367146279456, > 8388608, 70367199363104, 268780648, > > 268780648, 70367199388632, 70367200820192, 3, 0, > 70367199388648, 70368094386080, 70368094386136, 4001536, 70367199364120, > 70367146276672, -3187653564, > > 0 <repeats 42 times>}, mask_was_saved = 0}}, priv = {pad > {0x0, 0x0, 0x0, 0x0}, data = {prev = 0x0, cleanup = 0x0, canceltype = 0}}} > > not_first_call = <optimized out> > > pagesize_m1 = <optimized out> > > sp = <optimized out> > > freesize = <optimized out> > > __PRETTY_FUNCTION__ = "start_thread" > > #4 0x00003fffa3de60fc in .__clone () at > ../sysdeps/unix/sysv/linux/powerpc/powerpc64/clone.S:96 > > No locals. > > > > Thread 1 (LWP 1734): > > #0 0x00003fffa3d643b0 in _IO_unbuffer_all () at genops.c:960 > > __result = 0 > > __self = 0x3fffa1c20160 > > cnt = 1 > > fp = 0x3fff98001f00 > > #1 _IO_cleanup () at genops.c:1020 > > result = 0 > > #2 0x00003fffa3d1ed00 in __run_exit_handlers (status=<optimized out>, > listp=<optimized out>, run_list_atexit=run_list_atexit at entry=true) at > exit.c:95 > > ptr = 0x3fffa3e75000 > <__elf_set___libc_atexit_element__IO_cleanup__> > > #3 0x00003fffa3d1ee1c in __GI_exit (status=<optimized out>) at exit.c:104 > > No locals. > > #4 0x000000001000984c in cleanup_and_exit (signum=<optimized out>) at > glusterfsd.c:1295 > > ctx = <optimized out> > > trav = <optimized out> > > __FUNCTION__ = <error reading variable __FUNCTION__ (Cannot access > memory at address 0x10010e38)> > > #5 0x0000000010009a64 in glusterfs_sigwaiter (arg=<optimized out>) at > glusterfsd.c:2016 > > set = {__val = {18947, 0 <repeats 15 times>}} > > ret = <optimized out> > > sig = 15 > > #6 0x00003fffa3e9bb30 in start_thread (arg=0x3fffa1c20160) at > pthread_create.c:462 > > pd = 0x3fffa1c20160 > > now = <optimized out> > > unwind_buf = {cancel_jmp_buf = {{jmp_buf = {2380233324740058014, > 70367199403008, 2380233324703897146, 0, 0, 70367154671616, 70367163056672, > 8388608, 70367199363104, > > 70368094386576, 268595776, 70367199388632, 268588656, 3, > 0, 70367199388648, 70368094386224, 70368094386280, 4001536, 70367199364120, > 70367163053888, -3187653564, > > 0 <repeats 42 times>}, mask_was_saved = 0}}, priv = {pad > {0x0, 0x0, 0x0, 0x0}, data = {prev = 0x0, cleanup = 0x0, canceltype = 0}}} > > not_first_call = <optimized out> > > pagesize_m1 = <optimized out> > > sp = <optimized out> > > freesize = <optimized out> > > ---Type <return> to continue, or q <return> to quit--- > > __PRETTY_FUNCTION__ = "start_thread" > > #7 0x00003fffa3de60fc in .__clone () at > ../sysdeps/unix/sysv/linux/powerpc/powerpc64/clone.S:96 > > No locals. > > (gdb) > > (gdb) > > On Fri, Sep 21, 2018 at 3:32 PM ABHISHEK PALIWAL <abhishpaliwal at gmail.com> > wrote: > >> Hi Sanju, >> >> Here is the output of 't a a bt' >> >> (gdb) t a a bt >> >> Thread 7 (LWP 444): >> #0 0x00003fff7a4d4ccc in __pthread_cond_timedwait (cond=0x10059a98, >> mutex=0x10059a70, abstime=0x3fff77a50670) at pthread_cond_timedwait.c:198 >> >> #1 0x00003fff7a5f1e74 in syncenv_task (proc=0x10053eb0) at syncop.c:607 >> #2 0x00003fff7a5f2cdc in syncenv_processor (thdata=0x10053eb0) at >> syncop.c:699 >> #3 0x00003fff7a4ccb30 in start_thread (arg=0x3fff77a51160) at >> pthread_create.c:462 >> #4 0x00003fff7a4170fc in .__clone () at >> ../sysdeps/unix/sysv/linux/powerpc/powerpc64/clone.S:96 >> >> Thread 6 (LWP 447): >> #0 0x00003fff7a4d8e38 in __lll_lock_wait (futex=0x100272a8, >> private=<optimized out>) at lowlevellock.c:46 >> #1 0x00003fff7a4cff64 in __GI___pthread_mutex_lock (mutex=0x100272a8) at >> ../nptl/pthread_mutex_lock.c:81 >> #2 0x00003fff7a59de8c in _gf_msg (domain=0x3fff70006c90 >> "c_glusterfs-client-1", file=0x3fff76165de0 "client.c", >> function=0x3fff76165cd8 <__FUNCTION__.18849> "notify", >> line=<optimized out>, level=<optimized out>, errnum=<optimized out>, >> trace=<optimized out>, msgid=114020, >> fmt=0x3fff76166350 "parent translators are ready, attempting connect >> on transport") at logging.c:2058 >> #3 0x00003fff761394ac in notify (this=0x3fff700061f0, event=<optimized >> out>, data=0x3fff70008c50) at client.c:2116 >> #4 0x00003fff7a599ca0 in xlator_notify (xl=0x3fff700061f0, >> event=<optimized out>, data=<optimized out>) at xlator.c:491 >> #5 0x00003fff7a5b8700 in default_notify (this=0x3fff70008c50, >> event=<optimized out>, data=<optimized out>) at defaults.c:2302 >> #6 0x00003fff760fa22c in afr_notify (this=0x3fff70008c50, event=1, >> data=0x3fff7000a4c0, data2=<optimized out>) at afr-common.c:3967 >> #7 0x00003fff76105994 in notify (this=<optimized out>, event=<optimized >> out>, data=<optimized out>) at afr.c:38 >> #8 0x00003fff7a599ca0 in xlator_notify (xl=0x3fff70008c50, >> event=<optimized out>, data=<optimized out>) at xlator.c:491 >> #9 0x00003fff7a5b8700 in default_notify (this=0x3fff7000a4c0, >> event=<optimized out>, data=<optimized out>) at defaults.c:2302 >> #10 0x00003fff7609c1e4 in notify (this=<optimized out>, event=<optimized >> out>, data=<optimized out>) at io-stats.c:3064 >> #11 0x00003fff7a599ca0 in xlator_notify (xl=0x3fff7000a4c0, >> event=<optimized out>, data=<optimized out>) at xlator.c:491 >> #12 0x00003fff7a5ee560 in glusterfs_graph_parent_up (graph=<optimized >> out>) at graph.c:440 >> #13 0x00003fff7a5eeb90 in glusterfs_graph_activate (graph=0x3fff70000af0, >> ctx=0x10027010) at graph.c:688 >> #14 0x000000001000a49c in glusterfs_process_volfp (ctx=0x10027010, >> fp=0x3fff70001cd0) at glusterfsd.c:2221 >> #15 0x000000001000fd08 in mgmt_getspec_cbk (req=<optimized out>, >> iov=<optimized out>, count=<optimized out>, myframe=0x3fff7921b06c) at >> glusterfsd-mgmt.c:1561 >> #16 0x00003fff7a552ec4 in rpc_clnt_handle_reply (clnt=0x10089020, >> pollin=0x3fff70001760) at rpc-clnt.c:775 >> #17 0x00003fff7a5533d0 in rpc_clnt_notify (trans=<optimized out>, >> mydata=0x10089050, event=<optimized out>, data=<optimized out>) at >> rpc-clnt.c:933 >> #18 0x00003fff7a54e4fc in rpc_transport_notify (this=<optimized out>, >> event=<optimized out>, data=<optimized out>) at rpc-transport.c:546 >> #19 0x00003fff76a32d44 in socket_event_poll_in (this=this at entry=0x1008ab80) >> at socket.c:2236 >> #20 0x00003fff76a3589c in socket_event_handler (fd=<optimized out>, >> idx=<optimized out>, data=0x1008ab80, poll_in=<optimized out>, >> poll_out=<optimized out>, >> ---Type <return> to continue, or q <return> to quit--- >> poll_err=<optimized out>) at socket.c:2349 >> #21 0x00003fff7a613874 in event_dispatch_epoll_handler >> (event=0x3fff7697e6a0, event_pool=0x10045bc0) at event-epoll.c:575 >> #22 event_dispatch_epoll_worker (data=0x1008bd50) at event-epoll.c:678 >> #23 0x00003fff7a4ccb30 in start_thread (arg=0x3fff7697f160) at >> pthread_create.c:462 >> #24 0x00003fff7a4170fc in .__clone () at >> ../sysdeps/unix/sysv/linux/powerpc/powerpc64/clone.S:96 >> >> Thread 5 (LWP 453): >> #0 0x00003fff7a4d8e38 in __lll_lock_wait (futex=0x100272a8, >> private=<optimized out>) at lowlevellock.c:46 >> #1 0x00003fff7a4cff64 in __GI___pthread_mutex_lock (mutex=0x100272a8) at >> ../nptl/pthread_mutex_lock.c:81 >> #2 0x00003fff7a59de8c in _gf_msg (domain=0x3fff7a63ae38 "epoll", >> file=0x3fff7a63ae28 "event-epoll.c", >> function=0x3fff7a63adb8 <__FUNCTION__.8510> >> "event_dispatch_epoll_worker", line=<optimized out>, level=<optimized out>, >> errnum=<optimized out>, >> trace=<optimized out>, msgid=101190, fmt=0x3fff7a63af48 "Started >> thread with index %d") at logging.c:2058 >> #3 0x00003fff7a61365c in event_dispatch_epoll_worker >> (data=0x3fff7002ffd0) at event-epoll.c:631 >> #4 0x00003fff7a4ccb30 in start_thread (arg=0x3fff759ae160) at >> pthread_create.c:462 >> #5 0x00003fff7a4170fc in .__clone () at >> ../sysdeps/unix/sysv/linux/powerpc/powerpc64/clone.S:96 >> >> Thread 4 (LWP 441): >> #0 0x00003fff7a4ce084 in pthread_join (threadid=70366438879584, >> thread_return=0x0) at pthread_join.c:90 >> #1 0x00003fff7a613ca0 in event_dispatch_epoll (event_pool=0x10045bc0) at >> event-epoll.c:762 >> #2 0x00003fff7a5dc7e4 in event_dispatch (event_pool=<optimized out>) at >> event.c:128 >> #3 0x0000000010005ea8 in main (argc=<optimized out>, argv=<optimized >> out>) at glusterfsd.c:2380 >> #4 0x00003fff7a32f318 in generic_start_main (main=0x10025718 <main>, >> argc=<optimized out>, argv=0x3fffd839d7a8, auxvec=0x3fffd839d838, >> init=<optimized out>, >> rtld_fini=<optimized out>, stack_end=<optimized out>, fini=<optimized >> out>) at ../csu/libc-start.c:289 >> #5 0x00003fff7a32f54c in __libc_start_main (argc=<optimized out>, >> argv=<optimized out>, ev=<optimized out>, auxvec=<optimized out>, >> rtld_fini=<optimized out>, >> stinfo=<optimized out>, stack_on_entry=<optimized out>) at >> ../sysdeps/unix/sysv/linux/powerpc/libc-start.c:93 >> #6 0x0000000000000000 in ?? () >> >> Thread 3 (LWP 442): >> #0 0x00003fff7a4da150 in .__nanosleep () at >> ../sysdeps/unix/syscall-template.S:84 >> #1 0x00003fff7a5c4f28 in gf_timer_proc (ctx=0x10027010) at timer.c:205 >> #2 0x00003fff7a4ccb30 in start_thread (arg=0x3fff78a51160) at >> pthread_create.c:462 >> #3 0x00003fff7a4170fc in .__clone () at >> ../sysdeps/unix/sysv/linux/powerpc/powerpc64/clone.S:96 >> ---Type <return> to continue, or q <return> to quit--- >> >> Thread 2 (LWP 445): >> #0 0x00003fff7a4d4ccc in __pthread_cond_timedwait (cond=0x10059a98, >> mutex=0x10059a70, abstime=0x3fff77250670) at pthread_cond_timedwait.c:198 >> >> #1 0x00003fff7a5f1e74 in syncenv_task (proc=0x10054468) at syncop.c:607 >> #2 0 >> >-- Regards Abhishek Paliwal -------------- next part -------------- An HTML attachment was scrubbed... URL: <http://lists.gluster.org/pipermail/gluster-users/attachments/20180924/94f2afd9/attachment.html>