I've started seeing lots of crashes recently after having a stable
gluster for a couple of years. I upgraded from 9.6 to 10.3 in hopes of
more stability and got 3 crashes yesterday. All 3 seemed to have the
rpc_transport_unref line near the top, not sure what's going on, but
it's making the filesystem unusable:
Crash1
Program terminated with signal SIGBUS, Bus error.
#0 0x00007f0d66b4e7aa in __gf_free (free_ptr=0x7f0d58153678) at mem-pool.c:363
363 mem-pool.c: No such file or directory.
[Current thread is 1 (Thread 0x7f0d53fff700 (LWP 1664975))]
(gdb) where
#0 0x00007f0d66b4e7aa in __gf_free (free_ptr=0x7f0d58153678) at mem-pool.c:363
#1 __gf_free (free_ptr=0x7f0d58153678) at mem-pool.c:332
#2 0x00007f0d66ad04fa in rpc_transport_unref () from
/usr/lib/x86_64-linux-gnu/libgfrpc.so.0
#3 0x00007f0d66b7a71d in event_dispatch_epoll_handler
(event=0x7f0d53ffe054, event_pool=0x556c97385518) at event-epoll.c:638
#4 event_dispatch_epoll_worker (data=0x7f0d54006928) at event-epoll.c:749
#5 0x00007f0d66a90ea7 in start_thread (arg=<optimized out>) at
pthread_create.c:477
#6 0x00007f0d669b0a2f in clone () at
../sysdeps/unix/sysv/linux/x86_64/clone.S:95
Crash2:
Core was generated by /usr/sbin/glusterfsd
Program terminated with signal SIGSEGV, Segmentation fault.
#0 0x0000000000000000 in ?? ()
[Current thread is 1 (Thread 0x7f26fc1bc700 (LWP 1731799))]
(gdb) where
#0 0x0000000000000000 in ?? ()
#1 0x00007f270957f4a5 in rpc_transport_unref
(this=this at entry=0x7f26f40a0ad8) at rpc-transport.c:501
#2 0x00007f26feec3f4b in server_process_event_upcall
(this=this at entry=0x7f26f002b0f8, data=data at entry=0x7f26fc1b9a70) at
server.c:1499
#3 0x00007f26feec4964 in server_notify (this=0x7f26f002b0f8,
event=19, data=0x7f26fc1b9a70) at server.c:1620
#4 0x00007f27095cc244 in xlator_notify (xl=0x7f26f002b0f8, event=19,
data=0x7f26fc1b9a70) at xlator.c:711
#5 0x00007f270965df2b in default_notify (this=0x7f26f0028fd8,
event=<optimized out>, data=0x7f26fc1b9a70) at defaults.c:3414
#6 0x00007f26fef95b09 in ?? () from
/usr/lib/x86_64-linux-gnu/glusterfs/10.3/xlator/debug/io-stats.so
#7 0x00007f27095cc244 in xlator_notify (xl=0x7f26f0028fd8, event=19,
data=0x7f26fc1b9a70) at xlator.c:711
#8 0x00007f270965df2b in default_notify
(this=this at entry=0x7f26f00272c8, event=event at entry=19,
data=data at entry=0x7f26fc1b9a70) at defaults.c:3414
#9 0x00007f26fefc27ce in notify (this=0x7f26f00272c8, event=19,
data=0x7f26fc1b9a70) at quota.c:5017
#10 0x00007f27095cc244 in xlator_notify (xl=0x7f26f00272c8, event=19,
data=0x7f26fc1b9a70) at xlator.c:711
#11 0x00007f270965df2b in default_notify
(this=this at entry=0x7f26f0025548, event=event at entry=19,
data=data at entry=0x7f26fc1b9a70) at defaults.c:3414
#12 0x00007f26fefe47b1 in notify (this=0x7f26f0025548, event=19,
data=0x7f26fc1b9a70) at index.c:2664
#13 0x00007f27095cc244 in xlator_notify (xl=0x7f26f0025548, event=19,
data=0x7f26fc1b9a70) at xlator.c:711
#14 0x00007f270965df2b in default_notify
(this=this at entry=0x7f26f00239e8, event=event at entry=19,
data=data at entry=0x7f26fc1b9a70) at defaults.c:3414
#15 0x00007f26feff68c0 in notify (this=0x7f26f00239e8, event=19,
data=0x7f26fc1b9a70) at barrier.c:516
#16 0x00007f27095cc244 in xlator_notify (xl=0x7f26f00239e8, event=19,
data=0x7f26fc1b9a70) at xlator.c:711
#17 0x00007f270965df2b in default_notify (this=0x7f26f0021878,
event=<optimized out>, data=0x7f26fc1b9a70) at defaults.c:3414
#18 0x00007f27095cc244 in xlator_notify (xl=0x7f26f0021878, event=19,
data=0x7f26fc1b9a70) at xlator.c:711
#19 0x00007f270965df2b in default_notify (this=0x7f26f001fe28,
event=<optimized out>, data=0x7f26fc1b9a70) at defaults.c:3414
#20 0x00007f27095cc244 in xlator_notify (xl=0x7f26f001fe28, event=19,
data=0x7f26fc1b9a70) at xlator.c:711
#21 0x00007f270965df2b in default_notify
(this=this at entry=0x7f26f001e428, event=event at entry=19,
data=data at entry=0x7f26fc1b9a70) at defaults.c:3414
#22 0x00007f2704033efd in notify (this=0x7f26f001e428, event=19,
data=0x7f26fc1b9a70) at io-threads.c:1339
#23 0x00007f27095cc244 in xlator_notify (xl=0x7f26f001e428, event=19,
data=0x7f26fc1b9a70) at xlator.c:711
#24 0x00007f270965df2b in default_notify
(this=this at entry=0x7f26f001c8c8, event=event at entry=19,
data=data at entry=0x7f26fc1b9a70) at defaults.c:3414
#25 0x00007f270404ccc1 in notify (event=<optimized out>,
data=0x7f26fc1b9a70, this=0x7f26f001c8c8) at upcall.c:2368
#26 notify (this=0x7f26f001c8c8, event=<optimized out>,
data=0x7f26fc1b9a70) at upcall.c:2355
#27 0x00007f270404daa4 in upcall_client_cache_invalidate
(this=this at entry=0x7f26f001c8c8, gfid=gfid at entry=0x7f26f82dad34
"y-\300b\261;H6\221\306\003\031\307\306", <incomplete sequence
\342>,
up_client_entry=up_client_entry at entry=0x7f2680327d48,
flags=flags at entry=24,
stbuf=stbuf at entry=0x7f26fc1ba0d0, p_stbuf=p_stbuf at entry=0x0,
oldp_stbuf=0x0, xattr=0x0, now=1670297728) at upcall-internal.c:632
#28 0x00007f270405233c in upcall_cache_invalidate
(frame=0x7f2684be2738, this=0x7f26f001c8c8, client=0x7f26f022e078,
inode=<optimized out>, flags=24, stbuf=0x7f26fc1ba0d0, p_stbuf=0x0,
oldp_stbuf=0x0, xattr=0x0) at upcall-internal.c:566
#29 0x00007f2704041835 in upcall_cache_invalidate (xattr=0x0,
oldp_stbuf=0x0, p_stbuf=0x0, stbuf=0x7f26fc1ba0d0, flags=24,
inode=<optimized out>, client=<optimized out>, this=<optimized
out>,
frame=0x7f2684be2738) at upcall.c:91
#30 up_writev_cbk (frame=frame at entry=0x7f2684be2738, cookie=<optimized
out>, this=<optimized out>, op_ret=op_ret at entry=125683,
op_errno=op_errno at entry=0, prebuf=prebuf at entry=0x7f26fc1ba170,
postbuf=0x7f26fc1ba0d0, xdata=0x7f2684c1fe18) at upcall.c:98
#31 0x00007f27040618c5 in leases_writev_cbk
(frame=frame at entry=0x7f2684ff74f8, cookie=<optimized out>,
this=<optimized out>, op_ret=op_ret at entry=125683,
op_errno=op_errno at entry=0, prebuf=prebuf at entry=0x7f26fc1ba170,
postbuf=0x7f26fc1ba0d0, xdata=0x7f2684c1fe18)
at leases.c:97
#32 0x00007f27040a3bb2 in pl_writev_cbk
(frame=frame at entry=0x7f2684fc9a88, cookie=<optimized out>,
this=<optimized out>, op_ret=op_ret at entry=125683,
op_errno=op_errno at entry=0, prebuf=prebuf at entry=0x7f26fc1ba170,
postbuf=0x7f26fc1ba0d0, xdata=0x7f2684c1fe18)
at posix.c:2063
#33 0x00007f2709649aa5 in default_writev_cbk (frame=0x7f2685108e48,
cookie=<optimized out>, this=<optimized out>, op_ret=125683,
op_errno=0, prebuf=0x7f26fc1ba170, postbuf=0x7f26fc1ba0d0,
xdata=0x7f2684c1fe18) at defaults.c:1192
#34 0x00007f270410cc3f in changelog_writev_cbk (frame=0x7f26849d2eb8,
cookie=<optimized out>, this=<optimized out>, op_ret=125683,
op_errno=0, prebuf=0x7f26fc1ba170, postbuf=0x7f26fc1ba0d0,
xdata=0x7f2684c1fe18) at changelog.c:1736
#35 0x00007f27041b0498 in posix_writev (frame=<optimized out>,
this=<optimized out>, fd=<optimized out>, vector=0x7f26e07400a8,
count=1, offset=6112485950, flags=32769, iobref=0x7f26e81e5808,
xdata=0x7f26e1e84278) at posix-inode-fd-ops.c:2162
#36 0x00007f2709658762 in default_writev
(frame=frame at entry=0x7f2685c4cc88, this=<optimized out>,
fd=fd at entry=0x7f26e1607f68, vector=vector at entry=0x7f26e07400a8,
count=count at entry=1, off=off at entry=6112485950, flags=32769,
iobref=0x7f26e81e5808, xdata=0x7f26e1e84278)
at defaults.c:2550
#37 0x00007f27041155be in changelog_writev (frame=0x7f26849d2eb8,
this=0x7f26f0010808, fd=fd at entry=0x7f26e1607f68,
vector=vector at entry=0x7f26e07400a8, count=count at entry=1,
offset=offset at entry=6112485950, flags=32769, iobref=0x7f26e81e5808,
xdata=0x7f26e1e84278)
at changelog.c:1763
#38 0x00007f27040f4aa0 in br_stub_writev (frame=0x7f2685108e48,
this=0x7f26f0012738, fd=0x7f26e1607f68, vector=0x7f26e07400a8,
count=1, offset=6112485950, flags=32769, iobref=0x7f26e81e5808,
xdata=0x7f26e1e84278) at bit-rot-stub.c:2112
#39 0x00007f2709658762 in default_writev
(frame=frame at entry=0x7f2685108e48, this=<optimized out>,
fd=fd at entry=0x7f26e1607f68, vector=vector at entry=0x7f26e07400a8,
count=count at entry=1, off=off at entry=6112485950, flags=32769,
iobref=0x7f26e81e5808, xdata=0x7f26e1e84278)
at defaults.c:2550
#40 0x00007f27040a564f in pl_writev (frame=0x7f2684fc9a88,
this=0x7f26f0015ca8, fd=0x7f26e1607f68, vector=0x7f26e07400a8,
count=1, offset=6112485950, flags=32769, iobref=0x7f26e81e5808,
xdata=0x7f26e1e84278) at posix.c:2412
#41 0x00007f2704088ae1 in worm_writev (frame=0x7f2684fc9a88,
this=<optimized out>, fd=0x7f26e1607f68, vector=0x7f26e07400a8,
count=1, offset=6112485950, flags=32769, iobref=0x7f26e81e5808,
xdata=0x7f26e1e84278) at worm.c:429
#42 0x00007f270407cfa7 in ro_writev (frame=0x7f2684fc9a88,
this=<optimized out>, fd=fd at entry=0x7f26e1607f68,
vector=vector at entry=0x7f26e07400a8, count=count at entry=1,
off=off at entry=6112485950, flags=32769, iobref=0x7f26e81e5808,
xdata=0x7f26e1e84278)
at read-only-common.c:374
#43 0x00007f2704064d76 in leases_writev
(frame=frame at entry=0x7f2684ff74f8, this=0x7f26f001ae48,
fd=fd at entry=0x7f26e1607f68, vector=vector at entry=0x7f26e07400a8,
count=count at entry=1, off=off at entry=6112485950, flags=32769,
iobref=0x7f26e81e5808, xdata=0x7f26e1e84278)
at leases.c:132
#44 0x00007f27040462f0 in up_writev (frame=frame at entry=0x7f2684be2738,
this=0x7f26f001c8c8, fd=fd at entry=0x7f26e1607f68,
vector=vector at entry=0x7f26e07400a8, count=count at entry=1,
off=off at entry=6112485950, flags=32769, iobref=0x7f26e81e5808,
xdata=0x7f26e1e84278)
at upcall.c:124
#45 0x00007f2709652253 in default_writev_resume (frame=0x7f26e098c378,
this=0x7f26f001e428, fd=0x7f26e1607f68, vector=0x7f26e07400a8,
count=1, off=6112485950, flags=32769, iobref=0x7f26e81e5808,
xdata=0x7f26e1e84278) at defaults.c:1831
#46 0x00007f27095e58a8 in call_resume_wind (stub=<optimized out>) at
call-stub.c:1919
#47 0x00007f27095ff465 in call_resume (stub=stub at entry=0x7f26e0055368)
at call-stub.c:2387
#48 0x00007f2704032d98 in iot_worker (data=0x7f26f0043b58) at io-threads.c:232
#49 0x00007f270953fea7 in start_thread (arg=<optimized out>) at
pthread_create.c:477
#50 0x00007f270945fa2f in clone () at
../sysdeps/unix/sysv/linux/x86_64/clone.S:95
crash3:
Core was generated by /usr/sbin/glusterfsd
Program terminated with signal SIGABRT, Aborted.
#0 __GI_raise (sig=sig at entry=6) at ../sysdeps/unix/sysv/linux/raise.c:50
50 ../sysdeps/unix/sysv/linux/raise.c: No such file or directory.
[Current thread is 1 (Thread 0x7f74bc6f4700 (LWP 1480543))]
(gdb) where
#0 __GI_raise (sig=sig at entry=6) at ../sysdeps/unix/sysv/linux/raise.c:50
#1 0x00007f74d9123537 in __GI_abort () at abort.c:79
#2 0x00007f74d917c768 in __libc_message
(action=action at entry=do_abort, fmt=fmt at entry=0x7f74d929a3a5
"%s\n") at
../sysdeps/posix/libc_fatal.c:155
#3 0x00007f74d9183a5a in malloc_printerr
(str=str at entry=0x7f74d929c748 "double free or corruption
(fasttop)")
at malloc.c:5347
#4 0x00007f74d9184d55 in _int_free (av=0x7f74c8000020,
p=0x7f74c813abe0, have_lock=0) at malloc.c:4266
#5 0x00007f74d931d4fa in rpc_transport_unref () from
/usr/lib/x86_64-linux-gnu/libgfrpc.so.0
#6 0x00007f74cec62f4b in server_process_event_upcall
(this=this at entry=0x7f74c802ae28, data=data at entry=0x7f74bc6f1a70) at
server.c:1499
#7 0x00007f74cec63964 in server_notify (this=0x7f74c802ae28,
event=19, data=0x7f74bc6f1a70) at server.c:1620
#8 0x00007f74d936a244 in xlator_notify (xl=0x7f74c802ae28, event=19,
data=0x7f74bc6f1a70) at xlator.c:711
#9 0x00007f74d93fbf2b in default_notify (this=0x7f74c8028d08,
event=<optimized out>, data=0x7f74bc6f1a70) at defaults.c:3414
#10 0x00007f74ced34b09 in ?? () from
/usr/lib/x86_64-linux-gnu/glusterfs/10.3/xlator/debug/io-stats.so
#11 0x00007f74d936a244 in xlator_notify (xl=0x7f74c8028d08, event=19,
data=0x7f74bc6f1a70) at xlator.c:711
#12 0x00007f74d93fbf2b in default_notify
(this=this at entry=0x7f74c8026ff8, event=event at entry=19,
data=data at entry=0x7f74bc6f1a70) at defaults.c:3414
#13 0x00007f74ced617ce in notify (this=0x7f74c8026ff8, event=19,
data=0x7f74bc6f1a70) at quota.c:5017
#14 0x00007f74d936a244 in xlator_notify (xl=0x7f74c8026ff8, event=19,
data=0x7f74bc6f1a70) at xlator.c:711
#15 0x00007f74d93fbf2b in default_notify
(this=this at entry=0x7f74c8025278, event=event at entry=19,
data=data at entry=0x7f74bc6f1a70) at defaults.c:3414
#16 0x00007f74ced837b1 in notify (this=0x7f74c8025278, event=19,
data=0x7f74bc6f1a70) at index.c:2664
#17 0x00007f74d936a244 in xlator_notify (xl=0x7f74c8025278, event=19,
data=0x7f74bc6f1a70) at xlator.c:711
#18 0x00007f74d93fbf2b in default_notify
(this=this at entry=0x7f74c8023718, event=event at entry=19,
data=data at entry=0x7f74bc6f1a70) at defaults.c:3414
#19 0x00007f74ced958c0 in notify (this=0x7f74c8023718, event=19,
data=0x7f74bc6f1a70) at barrier.c:516
#20 0x00007f74d936a244 in xlator_notify (xl=0x7f74c8023718, event=19,
data=0x7f74bc6f1a70) at xlator.c:711
#21 0x00007f74d93fbf2b in default_notify (this=0x7f74c80215a8,
event=<optimized out>, data=0x7f74bc6f1a70) at defaults.c:3414
#22 0x00007f74d936a244 in xlator_notify (xl=0x7f74c80215a8, event=19,
data=0x7f74bc6f1a70) at xlator.c:711
#23 0x00007f74d93fbf2b in default_notify (this=0x7f74c801fb58,
event=<optimized out>, data=0x7f74bc6f1a70) at defaults.c:3414
#24 0x00007f74d936a244 in xlator_notify (xl=0x7f74c801fb58, event=19,
data=0x7f74bc6f1a70) at xlator.c:711
#25 0x00007f74d93fbf2b in default_notify
(this=this at entry=0x7f74c801e158, event=event at entry=19,
data=data at entry=0x7f74bc6f1a70) at defaults.c:3414
#26 0x00007f74cedc9efd in notify (this=0x7f74c801e158, event=19,
data=0x7f74bc6f1a70) at io-threads.c:1339
#27 0x00007f74d936a244 in xlator_notify (xl=0x7f74c801e158, event=19,
data=0x7f74bc6f1a70) at xlator.c:711
#28 0x00007f74d93fbf2b in default_notify
(this=this at entry=0x7f74c801c5f8, event=event at entry=19,
data=data at entry=0x7f74bc6f1a70) at defaults.c:3414
#29 0x00007f74cede2cc1 in notify (event=<optimized out>,
data=0x7f74bc6f1a70, this=0x7f74c801c5f8) at upcall.c:2368
#30 notify (this=0x7f74c801c5f8, event=<optimized out>,
data=0x7f74bc6f1a70) at upcall.c:2355
#31 0x00007f74cede3aa4 in upcall_client_cache_invalidate
(this=this at entry=0x7f74c801c5f8, gfid=gfid at entry=0x7f74b1932c34
"\232\301!!\225nA\016\263\360g\335P\273q]",
up_client_entry=up_client_entry at entry=0x7f74b5831eb8,
flags=flags at entry=24,
stbuf=stbuf at entry=0x7f74bc6f20d0, p_stbuf=p_stbuf at entry=0x0,
oldp_stbuf=0x0, xattr=0x0, now=1670267716) at upcall-internal.c:632
#32 0x00007f74cede833c in upcall_cache_invalidate
(frame=0x7f7448e09e08, this=0x7f74c801c5f8, client=0x7f74c0044978,
inode=<optimized out>, flags=24, stbuf=0x7f74bc6f20d0, p_stbuf=0x0,
oldp_stbuf=0x0, xattr=0x0) at upcall-internal.c:566
#33 0x00007f74cedd7835 in upcall_cache_invalidate (xattr=0x0,
oldp_stbuf=0x0, p_stbuf=0x0, stbuf=0x7f74bc6f20d0, flags=24,
inode=<optimized out>, client=<optimized out>, this=<optimized
out>,
frame=0x7f7448e09e08) at upcall.c:91
#34 up_writev_cbk (frame=frame at entry=0x7f7448e09e08, cookie=<optimized
out>, this=<optimized out>, op_ret=op_ret at entry=131072,
op_errno=op_errno at entry=0, prebuf=prebuf at entry=0x7f74bc6f2170,
postbuf=0x7f74bc6f20d0, xdata=0x7f7448e0c318) at upcall.c:98
#35 0x00007f74cedf78c5 in leases_writev_cbk
(frame=frame at entry=0x7f7448e1a498, cookie=<optimized out>,
this=<optimized out>, op_ret=op_ret at entry=131072,
op_errno=op_errno at entry=0, prebuf=prebuf at entry=0x7f74bc6f2170,
postbuf=0x7f74bc6f20d0, xdata=0x7f7448e0c318)
at leases.c:97
#36 0x00007f74cee39bb2 in pl_writev_cbk
(frame=frame at entry=0x7f7448e1a828, cookie=<optimized out>,
this=<optimized out>, op_ret=op_ret at entry=131072,
op_errno=op_errno at entry=0, prebuf=prebuf at entry=0x7f74bc6f2170,
postbuf=0x7f74bc6f20d0, xdata=0x7f7448e0c318)
at posix.c:2063
#37 0x00007f74d93e7aa5 in default_writev_cbk (frame=0x7f7448e091b8,
cookie=<optimized out>, this=<optimized out>, op_ret=131072,
op_errno=0, prebuf=0x7f74bc6f2170, postbuf=0x7f74bc6f20d0,
xdata=0x7f7448e0c318) at defaults.c:1192
#38 0x00007f74ceea2c3f in changelog_writev_cbk (frame=0x7f7448e0e5f8,
cookie=<optimized out>, this=<optimized out>, op_ret=131072,
op_errno=0, prebuf=0x7f74bc6f2170, postbuf=0x7f74bc6f20d0,
xdata=0x7f7448e0c318) at changelog.c:1736
#39 0x00007f74cef3c498 in posix_writev (frame=<optimized out>,
this=<optimized out>, fd=<optimized out>, vector=0x7f74951a1458,
count=1, offset=3770941440, flags=32770, iobref=0x7f74c002a938,
xdata=0x7f749543be98) at posix-inode-fd-ops.c:2162
#40 0x00007f74d93f6762 in default_writev
(frame=frame at entry=0x7f7448e0e768, this=<optimized out>,
fd=fd at entry=0x7f74b17974a8, vector=vector at entry=0x7f74951a1458,
count=count at entry=1, off=off at entry=3770941440, flags=32770,
iobref=0x7f74c002a938, xdata=0x7f749543be98)
at defaults.c:2550
#41 0x00007f74ceeab5be in changelog_writev (frame=0x7f7448e0e5f8,
this=0x7f74c8010538, fd=fd at entry=0x7f74b17974a8,
vector=vector at entry=0x7f74951a1458, count=count at entry=1,
offset=offset at entry=3770941440, flags=32770, iobref=0x7f74c002a938,
xdata=0x7f749543be98)
at changelog.c:1763
#42 0x00007f74cee8aaa0 in br_stub_writev (frame=0x7f7448e091b8,
this=0x7f74c8012468, fd=0x7f74b17974a8, vector=0x7f74951a1458,
count=1, offset=3770941440, flags=32770, iobref=0x7f74c002a938,
xdata=0x7f749543be98) at bit-rot-stub.c:2112
#43 0x00007f74d93f6762 in default_writev
(frame=frame at entry=0x7f7448e091b8, this=<optimized out>,
fd=fd at entry=0x7f74b17974a8, vector=vector at entry=0x7f74951a1458,
count=count at entry=1, off=off at entry=3770941440, flags=32770,
iobref=0x7f74c002a938, xdata=0x7f749543be98)
at defaults.c:2550
#44 0x00007f74cee3b64f in pl_writev (frame=0x7f7448e1a828,
this=0x7f74c80159d8, fd=0x7f74b17974a8, vector=0x7f74951a1458,
count=1, offset=3770941440, flags=32770, iobref=0x7f74c002a938,
xdata=0x7f749543be98) at posix.c:2412
#45 0x00007f74cee1eae1 in worm_writev (frame=0x7f7448e1a828,
this=<optimized out>, fd=0x7f74b17974a8, vector=0x7f74951a1458,
count=1, offset=3770941440, flags=32770, iobref=0x7f74c002a938,
xdata=0x7f749543be98) at worm.c:429
#46 0x00007f74cee12fa7 in ro_writev (frame=0x7f7448e1a828,
this=<optimized out>, fd=fd at entry=0x7f74b17974a8,
vector=vector at entry=0x7f74951a1458, count=count at entry=1,
off=off at entry=3770941440, flags=32770, iobref=0x7f74c002a938,
xdata=0x7f749543be98)
at read-only-common.c:374
#47 0x00007f74cedfad76 in leases_writev
(frame=frame at entry=0x7f7448e1a498, this=0x7f74c801ab78,
fd=fd at entry=0x7f74b17974a8, vector=vector at entry=0x7f74951a1458,
count=count at entry=1, off=off at entry=3770941440, flags=32770,
iobref=0x7f74c002a938, xdata=0x7f749543be98)
at leases.c:132
#48 0x00007f74ceddc2f0 in up_writev (frame=frame at entry=0x7f7448e09e08,
this=0x7f74c801c5f8, fd=fd at entry=0x7f74b17974a8,
vector=vector at entry=0x7f74951a1458, count=count at entry=1,
off=off at entry=3770941440, flags=32770, iobref=0x7f74c002a938,
xdata=0x7f749543be98)
at upcall.c:124
#49 0x00007f74d93f0253 in default_writev_resume (frame=0x7f749542cdd8,
this=0x7f74c801e158, fd=0x7f74b17974a8, vector=0x7f74951a1458,
count=1, off=3770941440, flags=32770, iobref=0x7f74c002a938,
xdata=0x7f749543be98) at defaults.c:1831
#50 0x00007f74d93838a8 in call_resume_wind (stub=<optimized out>) at
call-stub.c:1919
#51 0x00007f74d939d465 in call_resume (stub=stub at entry=0x7f749543ebf8)
at call-stub.c:2387
#52 0x00007f74cedc8d98 in iot_worker (data=0x7f74c8043888) at io-threads.c:232
#53 0x00007f74d92ddea7 in start_thread (arg=<optimized out>) at
pthread_create.c:477
#54 0x00007f74d91fda2f in clone () at
../sysdeps/unix/sysv/linux/x86_64/clone.S:95