Hi.
I have a question.
GlusterFS hdd space can't be recovered automatically after glusterfs
is disk full and I delete files.
It will be recovered restarting glusterfsd.
I can find deleted file and glusterfsd do not relese the file I don't know
why.
Is this OK?
[version]
CentOS release 6.7 (Final)
Linux web2 2.6.32-573.3.1.el6.x86_64 #1 SMP Thu Aug 13 22:55:16 UTC
2015 x86_64 x86_64 x86_64 GNU/Linux
glusterfs-3.6.5-1.el6.x86_64
glusterfs-api-3.6.5-1.el6.x86_64
glusterfs-fuse-3.6.5-1.el6.x86_64
glusterfs-server-3.6.5-1.el6.x86_64
glusterfs-libs-3.6.5-1.el6.x86_64
glusterfs-cli-3.6.5-1.el6.x86_6
[test]
dd if=/dev/zero of=./test.dat bs=1M count=100000
dd: writing `./test.dat': Input/output error
dd: closing output file `./test.dat': No space left on device
[root at web2 www_virtual]# df
Filesystem 1K-blocks Used Available Use% Mounted on
/dev/xvda1 8124856 2853600 4851880 38% /
tmpfs 509256 0 509256 0% /dev/shm
/dev/xvdf1 101441468 101419972 0 100% /glusterfs/vol01
web2:/vol_replica_01 101441408 101420032 0 100% /mnt/glusterfs
[root at web2 www_virtual]# rm test.dat
rm: remove regular file `test.dat'? y
[root at web2 www_virtual]# sync
[root at web2 www_virtual]# df
Filesystem 1K-blocks Used Available Use% Mounted on
/dev/xvda1 8124856 2856744 4848736 38% /
tmpfs 509256 0 509256 0% /dev/shm
/dev/xvdf1 101441468 101419972 0 100% /glusterfs/vol01
web2:/vol_replica_01 101441408 101420032 0 100% /mnt/glusterfs
Glusterfs is still disk full.
The other glusterfs server is same.
[find the deleted file]
(server web2)
[root at web2 www_virtual]# ls -l /proc/*/fd/* | grep deleted
lr-x------ 1 root root 64 Sep 11 14:15 /proc/1753/fd/14 ->
/var/lib/glusterd/snaps/missed_snaps_list (deleted)
lrwx------ 1 root root 64 Sep 11 14:17 /proc/1775/fd/21 ->
/glusterfs/vol01/brick/.glusterfs/1f/c2/1fc2b7b4-ecd6-4eff-a874-962c2283823f
(deleted)
[root at web2 .glusterfs]# ps ax | grep 1775 | grep -v grep
1775 ? Ssl 5:43 /usr/sbin/glusterfsd -s web2 --volfile-id
vol_replica_01.web2.glusterfs-vol01-brick -p
/var/lib/glusterd/vols/vol_replica_01/run/web2-glusterfs-vol01-brick.pid
-S /var/run/1cf98ee59b5dff8cfd793b8ec39851db.socket --brick-name
/glusterfs/vol01/brick -l
/var/log/glusterfs/bricks/glusterfs-vol01-brick.log --xlator-option
*-posix.glusterd-uuid=029cf626-935f-4546-a8df-f9d79a6959da
--brick-port 49152 --xlator-option
vol_replica_01-server.listen-port=49152
[root at web2 www_virtual]# lsof -p 1775
COMMAND PID USER FD TYPE DEVICE SIZE/OFF NODE NAME
glusterfs 1775 root cwd DIR 202,1 4096 2 /
glusterfs 1775 root rtd DIR 202,1 4096 2 /
glusterfs 1775 root txt REG 202,1 78056 266837
/usr/sbin/glusterfsd
glusterfs 1775 root mem REG 202,1 8560 854
/usr/lib64/glusterfs/3.6.5/auth/login.so
glusterfs 1775 root mem REG 202,1 13248 853
/usr/lib64/glusterfs/3.6.5/auth/addr.so
glusterfs 1775 root mem REG 202,1 224920 4334
/usr/lib64/glusterfs/3.6.5/xlator/protocol/server.so
glusterfs 1775 root mem REG 202,1 118760 2421
/usr/lib64/glusterfs/3.6.5/xlator/debug/io-stats.so
glusterfs 1775 root mem REG 202,1 119688 2507
/usr/lib64/glusterfs/3.6.5/xlator/features/quota.so
glusterfs 1775 root mem REG 202,1 139784 2491
/usr/lib64/glusterfs/3.6.5/xlator/features/marker.so
glusterfs 1775 root mem REG 202,1 42304 2475
/usr/lib64/glusterfs/3.6.5/xlator/features/index.so
glusterfs 1775 root mem REG 202,1 34360 2447
/usr/lib64/glusterfs/3.6.5/xlator/features/barrier.so
glusterfs 1775 root mem REG 202,1 46936 2561
/usr/lib64/glusterfs/3.6.5/xlator/performance/io-threads.so
glusterfs 1775 root mem REG 202,1 107984 2476
/usr/lib64/glusterfs/3.6.5/xlator/features/locks.so
glusterfs 1775 root mem REG 202,1 58440 3940
/usr/lib64/glusterfs/3.6.5/xlator/system/posix-acl.so
glusterfs 1775 root mem REG 202,1 90880 266322
/lib64/libgcc_s-4.4.7-20120601.so.1
glusterfs 1775 root mem REG 202,1 96200 2460
/usr/lib64/glusterfs/3.6.5/xlator/features/changelog.so
glusterfs 1775 root mem REG 202,1 3944 272350
/lib64/libaio.so.1.0.1
glusterfs 1775 root mem REG 202,1 188376 4337
/usr/lib64/glusterfs/3.6.5/xlator/storage/posix.so
glusterfs 1775 root mem REG 202,1 65928 273938
/lib64/libnss_files-2.12.so
glusterfs 1775 root mem REG 202,1 122040 264857
/lib64/libselinux.so.1
glusterfs 1775 root mem REG 202,1 110960 273940
/lib64/libresolv-2.12.so
glusterfs 1775 root mem REG 202,1 10192 267134
/lib64/libkeyutils.so.1.3
glusterfs 1775 root mem REG 202,1 43728 267143
/lib64/libkrb5support.so.0.1
glusterfs 1775 root mem REG 202,1 174840 267139
/lib64/libk5crypto.so.3.1
glusterfs 1775 root mem REG 202,1 14664 265208
/lib64/libcom_err.so.2.1
glusterfs 1775 root mem REG 202,1 946048 267141
/lib64/libkrb5.so.3.3
glusterfs 1775 root mem REG 202,1 277704 267135
/lib64/libgssapi_krb5.so.2.2
glusterfs 1775 root mem REG 202,1 441256 280919
/usr/lib64/libssl.so.1.0.1e
glusterfs 1775 root mem REG 202,1 83384 2328
/usr/lib64/glusterfs/3.6.5/rpc-transport/socket.so
glusterfs 1775 root mem REG 202,1 99158576 281041
/usr/lib/locale/locale-archive
glusterfs 1775 root mem REG 202,1 88600 265291
/lib64/libz.so.1.2.3
glusterfs 1775 root mem REG 202,1 19536 273934
/lib64/libdl-2.12.so
glusterfs 1775 root mem REG 202,1 1921216 264860
/lib64/libc-2.12.so
glusterfs 1775 root mem REG 202,1 1963296 267216
/usr/lib64/libcrypto.so.1.0.1e
glusterfs 1775 root mem REG 202,1 142640 264887
/lib64/libpthread-2.12.so
glusterfs 1775 root mem REG 202,1 43880 273941
/lib64/librt-2.12.so
glusterfs 1775 root mem REG 202,1 95968 265199
/usr/lib64/libgfxdr.so.0.0.1
glusterfs 1775 root mem REG 202,1 107536 264893
/usr/lib64/libgfrpc.so.0.0.1
glusterfs 1775 root mem REG 202,1 633304 265202
/usr/lib64/libglusterfs.so.0.0.1
glusterfs 1775 root mem REG 202,1 154664 264855
/lib64/ld-2.12.so
glusterfs 1775 root mem REG 202,1 26060 265172
/usr/lib64/gconv/gconv-modules.cache
glusterfs 1775 root 0r CHR 1,3 0t0 3923 /dev/null
glusterfs 1775 root 1w CHR 1,3 0t0 3923 /dev/null
glusterfs 1775 root 2w CHR 1,3 0t0 3923 /dev/null
glusterfs 1775 root 3u REG 0,9 0 3919
[eventpoll]
glusterfs 1775 root 4u unix 0xffff88003d027440 0t0 146494 socket
glusterfs 1775 root 5w REG 202,1 28428 778
/var/log/glusterfs/bricks/glusterfs-vol01-brick.log
glusterfs 1775 root 6uW REG 202,1 5 687
/var/lib/glusterd/vols/vol_replica_01/run/web2-glusterfs-vol01-brick.pid
glusterfs 1775 root 7u unix 0xffff880037b470c0 0t0 146497
/var/run/1cf98ee59b5dff8cfd793b8ec39851db.socket
glusterfs 1775 root 8u IPv4 146668 0t0 TCP
web2:49152->web2:1018 (ESTABLISHED)
glusterfs 1775 root 9u IPv4 146505 0t0 TCP
web2:1020->web2:24007 (ESTABLISHED)
glusterfs 1775 root 10u IPv4 146729 0t0 TCP
web2:49152->web2:1009 (ESTABLISHED)
glusterfs 1775 root 11u IPv4 146608 0t0 TCP
*:49152 (LISTEN)
glusterfs 1775 root 12r CHR 1,9 0t0 3928
/dev/urandom
glusterfs 1775 root 13r DIR 202,81 4096 5505025
/glusterfs/vol01/brick
glusterfs 1775 root 14u unix 0xffff88003d027780 0t0 146737
/var/run/1cf98ee59b5dff8cfd793b8ec39851db.socket
glusterfs 1775 root 15u IPv4 146749 0t0 TCP
web2:49152->web1:1020 (ESTABLISHED)
glusterfs 1775 root 16u IPv4 146751 0t0 TCP
web2:49152->web1:1017 (ESTABLISHED)
glusterfs 1775 root 17u IPv4 146753 0t0 TCP
web2:49152->web1:1014 (ESTABLISHED)
glusterfs 1775 root 18u IPv4 146759 0t0 TCP
web2:49152->web2:1006 (ESTABLISHED)
glusterfs 1775 root 20r DIR 202,81 4096 5505028
/glusterfs/vol01/brick/.glusterfs/indices/xattrop
glusterfs 1775 root 21u REG 202,81 16349855744 5520418
/glusterfs/vol01/brick/.glusterfs/1f/c2/1fc2b7b4-ecd6-4eff-a874-962c2283823f
(deleted)
(server web1)
[root at web1 www_virtual]# df
Filesystem 1K-blocks Used Available Use% Mounted on
/dev/xvda1 8124856 5608396 2097084 73% /
tmpfs 960324 0 960324 0% /dev/shm
/dev/xvdf1 101441468 101420068 0 100% /glusterfs/vol01
web1:/vol_replica_01 101441408 101420032 0 100% /mnt/glusterfs
[root at web1 www_virtual]# ls -l /proc/*/fd/* | grep deleted
ls: cannot access /proc/2348/fd/255: No such file or directory
ls: cannot access /proc/2348/fd/3: No such file or directory
ls: cannot access /proc/self/fd/255: No such file or directory
ls: cannot access /proc/self/fd/3: No such file or directory
lr-x------ 1 root root 64 Sep 11 14:21 /proc/1061/fd/14 ->
/var/lib/glusterd/snaps/missed_snaps_list (deleted)
lrwx------ 1 root root 64 Sep 11 14:21 /proc/1436/fd/16 ->
/glusterfs/vol01/brick/.glusterfs/1f/c2/1fc2b7b4-ecd6-4eff-a874-962c2283823f
(deleted)
[root at web1 www_virtual]# ps ax | grep 1436
1436 ? Ssl 15:24 /usr/sbin/glusterfsd -s web1 --volfile-id
vol_replica_01.web1.glusterfs-vol01-brick -p
/var/lib/glusterd/vols/vol_replica_01/run/web1-glusterfs-vol01-brick.pid
-S /var/run/623572fd107c86f79cd247c24572d645.socket --brick-name
/glusterfs/vol01/brick -l
/var/log/glusterfs/bricks/glusterfs-vol01-brick.log --xlator-option
*-posix.glusterd-uuid=cd648272-ac40-47c0-bbda-f2bb4894480a
--brick-port 49152 --xlator-option
vol_replica_01-server.listen-port=49152
[root at web1 www_virtual]# lsof -p 1436
COMMAND PID USER FD TYPE DEVICE SIZE/OFF NODE NAME
glusterfs 1436 root cwd DIR 202,1 4096 2 /
glusterfs 1436 root rtd DIR 202,1 4096 2 /
glusterfs 1436 root txt REG 202,1 78056 272348
/usr/sbin/glusterfsd
glusterfs 1436 root mem REG 202,1 8560 405980
/usr/lib64/glusterfs/3.6.5/auth/login.so
glusterfs 1436 root mem REG 202,1 13248 405978
/usr/lib64/glusterfs/3.6.5/auth/addr.so
glusterfs 1436 root mem REG 202,1 224920 406066
/usr/lib64/glusterfs/3.6.5/xlator/protocol/server.so
glusterfs 1436 root mem REG 202,1 118760 406003
/usr/lib64/glusterfs/3.6.5/xlator/debug/io-stats.so
glusterfs 1436 root mem REG 202,1 119688 406028
/usr/lib64/glusterfs/3.6.5/xlator/features/quota.so
glusterfs 1436 root mem REG 202,1 139784 406022
/usr/lib64/glusterfs/3.6.5/xlator/features/marker.so
glusterfs 1436 root mem REG 202,1 42304 406020
/usr/lib64/glusterfs/3.6.5/xlator/features/index.so
glusterfs 1436 root mem REG 202,1 34360 406016
/usr/lib64/glusterfs/3.6.5/xlator/features/barrier.so
glusterfs 1436 root mem REG 202,1 46936 406037
/usr/lib64/glusterfs/3.6.5/xlator/performance/io-threads.so
glusterfs 1436 root mem REG 202,1 107984 406021
/usr/lib64/glusterfs/3.6.5/xlator/features/locks.so
glusterfs 1436 root mem REG 202,1 58440 406048
/usr/lib64/glusterfs/3.6.5/xlator/system/posix-acl.so
glusterfs 1436 root mem REG 202,1 90880 267449
/lib64/libgcc_s-4.4.7-20120601.so.1
glusterfs 1436 root mem REG 202,1 96200 406018
/usr/lib64/glusterfs/3.6.5/xlator/features/changelog.so
glusterfs 1436 root mem REG 202,1 3944 272159
/lib64/libaio.so.1.0.1
glusterfs 1436 root mem REG 202,1 188376 406069
/usr/lib64/glusterfs/3.6.5/xlator/storage/posix.so
glusterfs 1436 root mem REG 202,1 65928 275388
/lib64/libnss_files-2.12.so
glusterfs 1436 root mem REG 202,1 122040 264857
/lib64/libselinux.so.1
glusterfs 1436 root mem REG 202,1 110960 275392
/lib64/libresolv-2.12.so
glusterfs 1436 root mem REG 202,1 10192 267134
/lib64/libkeyutils.so.1.3
glusterfs 1436 root mem REG 202,1 43728 267143
/lib64/libkrb5support.so.0.1
glusterfs 1436 root mem REG 202,1 174840 267139
/lib64/libk5crypto.so.3.1
glusterfs 1436 root mem REG 202,1 14664 265208
/lib64/libcom_err.so.2.1
glusterfs 1436 root mem REG 202,1 946048 267141
/lib64/libkrb5.so.3.3
glusterfs 1436 root mem REG 202,1 277704 267135
/lib64/libgssapi_krb5.so.2.2
glusterfs 1436 root mem REG 202,1 441256 281908
/usr/lib64/libssl.so.1.0.1e
glusterfs 1436 root mem REG 202,1 83384 405982
/usr/lib64/glusterfs/3.6.5/rpc-transport/socket.so
glusterfs 1436 root mem REG 202,1 99158576 272725
/usr/lib/locale/locale-archive
glusterfs 1436 root mem REG 202,1 88600 265291
/lib64/libz.so.1.2.3
glusterfs 1436 root mem REG 202,1 19536 274985
/lib64/libdl-2.12.so
glusterfs 1436 root mem REG 202,1 1921216 264860
/lib64/libc-2.12.so
glusterfs 1436 root mem REG 202,1 1963296 267216
/usr/lib64/libcrypto.so.1.0.1e
glusterfs 1436 root mem REG 202,1 142640 264887
/lib64/libpthread-2.12.so
glusterfs 1436 root mem REG 202,1 43880 275393
/lib64/librt-2.12.so
glusterfs 1436 root mem REG 202,1 95968 272324
/usr/lib64/libgfxdr.so.0.0.1
glusterfs 1436 root mem REG 202,1 107536 272323
/usr/lib64/libgfrpc.so.0.0.1
glusterfs 1436 root mem REG 202,1 633304 272325
/usr/lib64/libglusterfs.so.0.0.1
glusterfs 1436 root mem REG 202,1 154664 264855
/lib64/ld-2.12.so
glusterfs 1436 root mem REG 202,1 26060 265172
/usr/lib64/gconv/gconv-modules.cache
glusterfs 1436 root 0r CHR 1,3 0t0 3923 /dev/null
glusterfs 1436 root 1w CHR 1,3 0t0 3923 /dev/null
glusterfs 1436 root 2w CHR 1,3 0t0 3923 /dev/null
glusterfs 1436 root 3u REG 0,9 0 3919
[eventpoll]
glusterfs 1436 root 4u unix 0xffff88007b2b23c0 0t0 75161 socket
glusterfs 1436 root 5w REG 202,1 13674 838
/var/log/glusterfs/bricks/glusterfs-vol01-brick.log
glusterfs 1436 root 6uW REG 202,1 5 401217
/var/lib/glusterd/vols/vol_replica_01/run/web1-glusterfs-vol01-brick.pid
glusterfs 1436 root 7u unix 0xffff88003799f0c0 0t0 75221
/var/run/623572fd107c86f79cd247c24572d645.socket
glusterfs 1436 root 8u IPv4 75365 0t0 TCP
web1:49152->web1:1016 (ESTABLISHED)
glusterfs 1436 root 9u IPv4 75231 0t0 TCP
web1:1018->web1:24007 (ESTABLISHED)
glusterfs 1436 root 10u IPv4 75397 0t0 TCP
web1:49152->web1:1009 (ESTABLISHED)
glusterfs 1436 root 11u IPv4 75275 0t0 TCP
*:49152 (LISTEN)
glusterfs 1436 root 12r CHR 1,9 0t0 3928
/dev/urandom
glusterfs 1436 root 13r DIR 202,81 4096 2621441
/glusterfs/vol01/brick
glusterfs 1436 root 14u IPv4 75401 0t0 TCP
web1:49152->web2:1015 (ESTABLISHED)
glusterfs 1436 root 16u REG 202,81 16349941760 2639508
/glusterfs/vol01/brick/.glusterfs/1f/c2/1fc2b7b4-ecd6-4eff-a874-962c2283823f
(deleted)
glusterfs 1436 root 17u unix 0xffff880079b85100 0t0 75422
/var/run/623572fd107c86f79cd247c24572d645.socket
glusterfs 1436 root 18u IPv4 75428 0t0 TCP
web1:49152->web1:1007 (ESTABLISHED)
glusterfs 1436 root 19u IPv4 92958 0t0 TCP
web1:49152->web2:1008 (ESTABLISHED)
glusterfs 1436 root 20u IPv4 92955 0t0 TCP
web1:49152->web2:1017 (ESTABLISHED)
[gluster status]
[root at web2 www_virtual]# gluster volume info
Volume Name: vol_replica_01
Type: Replicate
Volume ID: c6f5b52c-865f-4dc1-9377-ea81d933b859
Status: Started
Number of Bricks: 1 x 2 = 2
Transport-type: tcp
Bricks:
Brick1: web1:/glusterfs/vol01/brick
Brick2: web2:/glusterfs/vol01/brick
Options Reconfigured:
network.ping-timeout: 15
performance.cache-max-file-size: 100MB
[root at web1 www_virtual]# gluster volume info
Volume Name: vol_replica_01
Type: Replicate
Volume ID: c6f5b52c-865f-4dc1-9377-ea81d933b859
Status: Started
Number of Bricks: 1 x 2 = 2
Transport-type: tcp
Bricks:
Brick1: web1:/glusterfs/vol01/brick
Brick2: web2:/glusterfs/vol01/brick
Options Reconfigured:
network.ping-timeout: 15
performance.cache-max-file-size: 100MB
[gluster config]
/etc/glusterfs/glusterd.vol
volume management
type mgmt/glusterd
option working-directory /var/lib/glusterd
option transport-type socket
option transport.socket.keepalive-time 10
option transport.socket.keepalive-interval 2
option transport.socket.read-fail-log off
option ping-timeout 30
end-volume
Best,
Yasu
If you have an active snapshot, I expect the space will not be freed until you remove the snapshot. On 11 September 2015 at 01:44, Fujii Yasuhiro <eyes.works at gmail.com> wrote:> Hi. > > I have a question. > > GlusterFS hdd space can't be recovered automatically after glusterfs > is disk full and I delete files. > It will be recovered restarting glusterfsd. > I can find deleted file and glusterfsd do not relese the file I don't know > why. > Is this OK? > > [version] > CentOS release 6.7 (Final) > Linux web2 2.6.32-573.3.1.el6.x86_64 #1 SMP Thu Aug 13 22:55:16 UTC > 2015 x86_64 x86_64 x86_64 GNU/Linux > glusterfs-3.6.5-1.el6.x86_64 > glusterfs-api-3.6.5-1.el6.x86_64 > glusterfs-fuse-3.6.5-1.el6.x86_64 > glusterfs-server-3.6.5-1.el6.x86_64 > glusterfs-libs-3.6.5-1.el6.x86_64 > glusterfs-cli-3.6.5-1.el6.x86_6 > > [test] > dd if=/dev/zero of=./test.dat bs=1M count=100000 > dd: writing `./test.dat': Input/output error > dd: closing output file `./test.dat': No space left on device > > [root at web2 www_virtual]# df > Filesystem 1K-blocks Used Available Use% Mounted on > /dev/xvda1 8124856 2853600 4851880 38% / > tmpfs 509256 0 509256 0% /dev/shm > /dev/xvdf1 101441468 101419972 0 100% /glusterfs/vol01 > web2:/vol_replica_01 101441408 101420032 0 100% /mnt/glusterfs > > [root at web2 www_virtual]# rm test.dat > rm: remove regular file `test.dat'? y > > [root at web2 www_virtual]# sync > [root at web2 www_virtual]# df > Filesystem 1K-blocks Used Available Use% Mounted on > /dev/xvda1 8124856 2856744 4848736 38% / > tmpfs 509256 0 509256 0% /dev/shm > /dev/xvdf1 101441468 101419972 0 100% /glusterfs/vol01 > web2:/vol_replica_01 101441408 101420032 0 100% /mnt/glusterfs > > Glusterfs is still disk full. > The other glusterfs server is same. > > [find the deleted file] > (server web2) > [root at web2 www_virtual]# ls -l /proc/*/fd/* | grep deleted > lr-x------ 1 root root 64 Sep 11 14:15 /proc/1753/fd/14 -> > /var/lib/glusterd/snaps/missed_snaps_list (deleted) > lrwx------ 1 root root 64 Sep 11 14:17 /proc/1775/fd/21 -> > > /glusterfs/vol01/brick/.glusterfs/1f/c2/1fc2b7b4-ecd6-4eff-a874-962c2283823f > (deleted) > > [root at web2 .glusterfs]# ps ax | grep 1775 | grep -v grep > 1775 ? Ssl 5:43 /usr/sbin/glusterfsd -s web2 --volfile-id > vol_replica_01.web2.glusterfs-vol01-brick -p > /var/lib/glusterd/vols/vol_replica_01/run/web2-glusterfs-vol01-brick.pid > -S /var/run/1cf98ee59b5dff8cfd793b8ec39851db.socket --brick-name > /glusterfs/vol01/brick -l > /var/log/glusterfs/bricks/glusterfs-vol01-brick.log --xlator-option > *-posix.glusterd-uuid=029cf626-935f-4546-a8df-f9d79a6959da > --brick-port 49152 --xlator-option > vol_replica_01-server.listen-port=49152 > > [root at web2 www_virtual]# lsof -p 1775 > COMMAND PID USER FD TYPE DEVICE SIZE/OFF NODE NAME > glusterfs 1775 root cwd DIR 202,1 4096 2 / > glusterfs 1775 root rtd DIR 202,1 4096 2 / > glusterfs 1775 root txt REG 202,1 78056 266837 > /usr/sbin/glusterfsd > glusterfs 1775 root mem REG 202,1 8560 854 > /usr/lib64/glusterfs/3.6.5/auth/login.so > glusterfs 1775 root mem REG 202,1 13248 853 > /usr/lib64/glusterfs/3.6.5/auth/addr.so > glusterfs 1775 root mem REG 202,1 224920 4334 > /usr/lib64/glusterfs/3.6.5/xlator/protocol/server.so > glusterfs 1775 root mem REG 202,1 118760 2421 > /usr/lib64/glusterfs/3.6.5/xlator/debug/io-stats.so > glusterfs 1775 root mem REG 202,1 119688 2507 > /usr/lib64/glusterfs/3.6.5/xlator/features/quota.so > glusterfs 1775 root mem REG 202,1 139784 2491 > /usr/lib64/glusterfs/3.6.5/xlator/features/marker.so > glusterfs 1775 root mem REG 202,1 42304 2475 > /usr/lib64/glusterfs/3.6.5/xlator/features/index.so > glusterfs 1775 root mem REG 202,1 34360 2447 > /usr/lib64/glusterfs/3.6.5/xlator/features/barrier.so > glusterfs 1775 root mem REG 202,1 46936 2561 > /usr/lib64/glusterfs/3.6.5/xlator/performance/io-threads.so > glusterfs 1775 root mem REG 202,1 107984 2476 > /usr/lib64/glusterfs/3.6.5/xlator/features/locks.so > glusterfs 1775 root mem REG 202,1 58440 3940 > /usr/lib64/glusterfs/3.6.5/xlator/system/posix-acl.so > glusterfs 1775 root mem REG 202,1 90880 266322 > /lib64/libgcc_s-4.4.7-20120601.so.1 > glusterfs 1775 root mem REG 202,1 96200 2460 > /usr/lib64/glusterfs/3.6.5/xlator/features/changelog.so > glusterfs 1775 root mem REG 202,1 3944 272350 > /lib64/libaio.so.1.0.1 > glusterfs 1775 root mem REG 202,1 188376 4337 > /usr/lib64/glusterfs/3.6.5/xlator/storage/posix.so > glusterfs 1775 root mem REG 202,1 65928 273938 > /lib64/libnss_files-2.12.so > glusterfs 1775 root mem REG 202,1 122040 264857 > /lib64/libselinux.so.1 > glusterfs 1775 root mem REG 202,1 110960 273940 > /lib64/libresolv-2.12.so > glusterfs 1775 root mem REG 202,1 10192 267134 > /lib64/libkeyutils.so.1.3 > glusterfs 1775 root mem REG 202,1 43728 267143 > /lib64/libkrb5support.so.0.1 > glusterfs 1775 root mem REG 202,1 174840 267139 > /lib64/libk5crypto.so.3.1 > glusterfs 1775 root mem REG 202,1 14664 265208 > /lib64/libcom_err.so.2.1 > glusterfs 1775 root mem REG 202,1 946048 267141 > /lib64/libkrb5.so.3.3 > glusterfs 1775 root mem REG 202,1 277704 267135 > /lib64/libgssapi_krb5.so.2.2 > glusterfs 1775 root mem REG 202,1 441256 280919 > /usr/lib64/libssl.so.1.0.1e > glusterfs 1775 root mem REG 202,1 83384 2328 > /usr/lib64/glusterfs/3.6.5/rpc-transport/socket.so > glusterfs 1775 root mem REG 202,1 99158576 281041 > /usr/lib/locale/locale-archive > glusterfs 1775 root mem REG 202,1 88600 265291 > /lib64/libz.so.1.2.3 > glusterfs 1775 root mem REG 202,1 19536 273934 > /lib64/libdl-2.12.so > glusterfs 1775 root mem REG 202,1 1921216 264860 > /lib64/libc-2.12.so > glusterfs 1775 root mem REG 202,1 1963296 267216 > /usr/lib64/libcrypto.so.1.0.1e > glusterfs 1775 root mem REG 202,1 142640 264887 > /lib64/libpthread-2.12.so > glusterfs 1775 root mem REG 202,1 43880 273941 > /lib64/librt-2.12.so > glusterfs 1775 root mem REG 202,1 95968 265199 > /usr/lib64/libgfxdr.so.0.0.1 > glusterfs 1775 root mem REG 202,1 107536 264893 > /usr/lib64/libgfrpc.so.0.0.1 > glusterfs 1775 root mem REG 202,1 633304 265202 > /usr/lib64/libglusterfs.so.0.0.1 > glusterfs 1775 root mem REG 202,1 154664 264855 > /lib64/ld-2.12.so > glusterfs 1775 root mem REG 202,1 26060 265172 > /usr/lib64/gconv/gconv-modules.cache > glusterfs 1775 root 0r CHR 1,3 0t0 3923 > /dev/null > glusterfs 1775 root 1w CHR 1,3 0t0 3923 > /dev/null > glusterfs 1775 root 2w CHR 1,3 0t0 3923 > /dev/null > glusterfs 1775 root 3u REG 0,9 0 3919 > [eventpoll] > glusterfs 1775 root 4u unix 0xffff88003d027440 0t0 146494 > socket > glusterfs 1775 root 5w REG 202,1 28428 778 > /var/log/glusterfs/bricks/glusterfs-vol01-brick.log > glusterfs 1775 root 6uW REG 202,1 5 687 > /var/lib/glusterd/vols/vol_replica_01/run/web2-glusterfs-vol01-brick.pid > glusterfs 1775 root 7u unix 0xffff880037b470c0 0t0 146497 > /var/run/1cf98ee59b5dff8cfd793b8ec39851db.socket > glusterfs 1775 root 8u IPv4 146668 0t0 TCP > web2:49152->web2:1018 (ESTABLISHED) > glusterfs 1775 root 9u IPv4 146505 0t0 TCP > web2:1020->web2:24007 (ESTABLISHED) > glusterfs 1775 root 10u IPv4 146729 0t0 TCP > web2:49152->web2:1009 (ESTABLISHED) > glusterfs 1775 root 11u IPv4 146608 0t0 TCP > *:49152 (LISTEN) > glusterfs 1775 root 12r CHR 1,9 0t0 3928 > /dev/urandom > glusterfs 1775 root 13r DIR 202,81 4096 5505025 > /glusterfs/vol01/brick > glusterfs 1775 root 14u unix 0xffff88003d027780 0t0 146737 > /var/run/1cf98ee59b5dff8cfd793b8ec39851db.socket > glusterfs 1775 root 15u IPv4 146749 0t0 TCP > web2:49152->web1:1020 (ESTABLISHED) > glusterfs 1775 root 16u IPv4 146751 0t0 TCP > web2:49152->web1:1017 (ESTABLISHED) > glusterfs 1775 root 17u IPv4 146753 0t0 TCP > web2:49152->web1:1014 (ESTABLISHED) > glusterfs 1775 root 18u IPv4 146759 0t0 TCP > web2:49152->web2:1006 (ESTABLISHED) > glusterfs 1775 root 20r DIR 202,81 4096 5505028 > /glusterfs/vol01/brick/.glusterfs/indices/xattrop > glusterfs 1775 root 21u REG 202,81 16349855744 5520418 > > /glusterfs/vol01/brick/.glusterfs/1f/c2/1fc2b7b4-ecd6-4eff-a874-962c2283823f > (deleted) > > (server web1) > [root at web1 www_virtual]# df > Filesystem 1K-blocks Used Available Use% Mounted on > /dev/xvda1 8124856 5608396 2097084 73% / > tmpfs 960324 0 960324 0% /dev/shm > /dev/xvdf1 101441468 101420068 0 100% /glusterfs/vol01 > web1:/vol_replica_01 101441408 101420032 0 100% /mnt/glusterfs > > [root at web1 www_virtual]# ls -l /proc/*/fd/* | grep deleted > ls: cannot access /proc/2348/fd/255: No such file or directory > ls: cannot access /proc/2348/fd/3: No such file or directory > ls: cannot access /proc/self/fd/255: No such file or directory > ls: cannot access /proc/self/fd/3: No such file or directory > lr-x------ 1 root root 64 Sep 11 14:21 /proc/1061/fd/14 -> > /var/lib/glusterd/snaps/missed_snaps_list (deleted) > lrwx------ 1 root root 64 Sep 11 14:21 /proc/1436/fd/16 -> > > /glusterfs/vol01/brick/.glusterfs/1f/c2/1fc2b7b4-ecd6-4eff-a874-962c2283823f > (deleted) > > [root at web1 www_virtual]# ps ax | grep 1436 > 1436 ? Ssl 15:24 /usr/sbin/glusterfsd -s web1 --volfile-id > vol_replica_01.web1.glusterfs-vol01-brick -p > /var/lib/glusterd/vols/vol_replica_01/run/web1-glusterfs-vol01-brick.pid > -S /var/run/623572fd107c86f79cd247c24572d645.socket --brick-name > /glusterfs/vol01/brick -l > /var/log/glusterfs/bricks/glusterfs-vol01-brick.log --xlator-option > *-posix.glusterd-uuid=cd648272-ac40-47c0-bbda-f2bb4894480a > --brick-port 49152 --xlator-option > vol_replica_01-server.listen-port=49152 > > [root at web1 www_virtual]# lsof -p 1436 > COMMAND PID USER FD TYPE DEVICE SIZE/OFF NODE NAME > glusterfs 1436 root cwd DIR 202,1 4096 2 / > glusterfs 1436 root rtd DIR 202,1 4096 2 / > glusterfs 1436 root txt REG 202,1 78056 272348 > /usr/sbin/glusterfsd > glusterfs 1436 root mem REG 202,1 8560 405980 > /usr/lib64/glusterfs/3.6.5/auth/login.so > glusterfs 1436 root mem REG 202,1 13248 405978 > /usr/lib64/glusterfs/3.6.5/auth/addr.so > glusterfs 1436 root mem REG 202,1 224920 406066 > /usr/lib64/glusterfs/3.6.5/xlator/protocol/server.so > glusterfs 1436 root mem REG 202,1 118760 406003 > /usr/lib64/glusterfs/3.6.5/xlator/debug/io-stats.so > glusterfs 1436 root mem REG 202,1 119688 406028 > /usr/lib64/glusterfs/3.6.5/xlator/features/quota.so > glusterfs 1436 root mem REG 202,1 139784 406022 > /usr/lib64/glusterfs/3.6.5/xlator/features/marker.so > glusterfs 1436 root mem REG 202,1 42304 406020 > /usr/lib64/glusterfs/3.6.5/xlator/features/index.so > glusterfs 1436 root mem REG 202,1 34360 406016 > /usr/lib64/glusterfs/3.6.5/xlator/features/barrier.so > glusterfs 1436 root mem REG 202,1 46936 406037 > /usr/lib64/glusterfs/3.6.5/xlator/performance/io-threads.so > glusterfs 1436 root mem REG 202,1 107984 406021 > /usr/lib64/glusterfs/3.6.5/xlator/features/locks.so > glusterfs 1436 root mem REG 202,1 58440 406048 > /usr/lib64/glusterfs/3.6.5/xlator/system/posix-acl.so > glusterfs 1436 root mem REG 202,1 90880 267449 > /lib64/libgcc_s-4.4.7-20120601.so.1 > glusterfs 1436 root mem REG 202,1 96200 406018 > /usr/lib64/glusterfs/3.6.5/xlator/features/changelog.so > glusterfs 1436 root mem REG 202,1 3944 272159 > /lib64/libaio.so.1.0.1 > glusterfs 1436 root mem REG 202,1 188376 406069 > /usr/lib64/glusterfs/3.6.5/xlator/storage/posix.so > glusterfs 1436 root mem REG 202,1 65928 275388 > /lib64/libnss_files-2.12.so > glusterfs 1436 root mem REG 202,1 122040 264857 > /lib64/libselinux.so.1 > glusterfs 1436 root mem REG 202,1 110960 275392 > /lib64/libresolv-2.12.so > glusterfs 1436 root mem REG 202,1 10192 267134 > /lib64/libkeyutils.so.1.3 > glusterfs 1436 root mem REG 202,1 43728 267143 > /lib64/libkrb5support.so.0.1 > glusterfs 1436 root mem REG 202,1 174840 267139 > /lib64/libk5crypto.so.3.1 > glusterfs 1436 root mem REG 202,1 14664 265208 > /lib64/libcom_err.so.2.1 > glusterfs 1436 root mem REG 202,1 946048 267141 > /lib64/libkrb5.so.3.3 > glusterfs 1436 root mem REG 202,1 277704 267135 > /lib64/libgssapi_krb5.so.2.2 > glusterfs 1436 root mem REG 202,1 441256 281908 > /usr/lib64/libssl.so.1.0.1e > glusterfs 1436 root mem REG 202,1 83384 405982 > /usr/lib64/glusterfs/3.6.5/rpc-transport/socket.so > glusterfs 1436 root mem REG 202,1 99158576 272725 > /usr/lib/locale/locale-archive > glusterfs 1436 root mem REG 202,1 88600 265291 > /lib64/libz.so.1.2.3 > glusterfs 1436 root mem REG 202,1 19536 274985 > /lib64/libdl-2.12.so > glusterfs 1436 root mem REG 202,1 1921216 264860 > /lib64/libc-2.12.so > glusterfs 1436 root mem REG 202,1 1963296 267216 > /usr/lib64/libcrypto.so.1.0.1e > glusterfs 1436 root mem REG 202,1 142640 264887 > /lib64/libpthread-2.12.so > glusterfs 1436 root mem REG 202,1 43880 275393 > /lib64/librt-2.12.so > glusterfs 1436 root mem REG 202,1 95968 272324 > /usr/lib64/libgfxdr.so.0.0.1 > glusterfs 1436 root mem REG 202,1 107536 272323 > /usr/lib64/libgfrpc.so.0.0.1 > glusterfs 1436 root mem REG 202,1 633304 272325 > /usr/lib64/libglusterfs.so.0.0.1 > glusterfs 1436 root mem REG 202,1 154664 264855 > /lib64/ld-2.12.so > glusterfs 1436 root mem REG 202,1 26060 265172 > /usr/lib64/gconv/gconv-modules.cache > glusterfs 1436 root 0r CHR 1,3 0t0 3923 > /dev/null > glusterfs 1436 root 1w CHR 1,3 0t0 3923 > /dev/null > glusterfs 1436 root 2w CHR 1,3 0t0 3923 > /dev/null > glusterfs 1436 root 3u REG 0,9 0 3919 > [eventpoll] > glusterfs 1436 root 4u unix 0xffff88007b2b23c0 0t0 75161 > socket > glusterfs 1436 root 5w REG 202,1 13674 838 > /var/log/glusterfs/bricks/glusterfs-vol01-brick.log > glusterfs 1436 root 6uW REG 202,1 5 401217 > /var/lib/glusterd/vols/vol_replica_01/run/web1-glusterfs-vol01-brick.pid > glusterfs 1436 root 7u unix 0xffff88003799f0c0 0t0 75221 > /var/run/623572fd107c86f79cd247c24572d645.socket > glusterfs 1436 root 8u IPv4 75365 0t0 TCP > web1:49152->web1:1016 (ESTABLISHED) > glusterfs 1436 root 9u IPv4 75231 0t0 TCP > web1:1018->web1:24007 (ESTABLISHED) > glusterfs 1436 root 10u IPv4 75397 0t0 TCP > web1:49152->web1:1009 (ESTABLISHED) > glusterfs 1436 root 11u IPv4 75275 0t0 TCP > *:49152 (LISTEN) > glusterfs 1436 root 12r CHR 1,9 0t0 3928 > /dev/urandom > glusterfs 1436 root 13r DIR 202,81 4096 2621441 > /glusterfs/vol01/brick > glusterfs 1436 root 14u IPv4 75401 0t0 TCP > web1:49152->web2:1015 (ESTABLISHED) > glusterfs 1436 root 16u REG 202,81 16349941760 2639508 > > /glusterfs/vol01/brick/.glusterfs/1f/c2/1fc2b7b4-ecd6-4eff-a874-962c2283823f > (deleted) > glusterfs 1436 root 17u unix 0xffff880079b85100 0t0 75422 > /var/run/623572fd107c86f79cd247c24572d645.socket > glusterfs 1436 root 18u IPv4 75428 0t0 TCP > web1:49152->web1:1007 (ESTABLISHED) > glusterfs 1436 root 19u IPv4 92958 0t0 TCP > web1:49152->web2:1008 (ESTABLISHED) > glusterfs 1436 root 20u IPv4 92955 0t0 TCP > web1:49152->web2:1017 (ESTABLISHED) > > [gluster status] > > [root at web2 www_virtual]# gluster volume info > > Volume Name: vol_replica_01 > Type: Replicate > Volume ID: c6f5b52c-865f-4dc1-9377-ea81d933b859 > Status: Started > Number of Bricks: 1 x 2 = 2 > Transport-type: tcp > Bricks: > Brick1: web1:/glusterfs/vol01/brick > Brick2: web2:/glusterfs/vol01/brick > Options Reconfigured: > network.ping-timeout: 15 > performance.cache-max-file-size: 100MB > > [root at web1 www_virtual]# gluster volume info > > Volume Name: vol_replica_01 > Type: Replicate > Volume ID: c6f5b52c-865f-4dc1-9377-ea81d933b859 > Status: Started > Number of Bricks: 1 x 2 = 2 > Transport-type: tcp > Bricks: > Brick1: web1:/glusterfs/vol01/brick > Brick2: web2:/glusterfs/vol01/brick > Options Reconfigured: > network.ping-timeout: 15 > performance.cache-max-file-size: 100MB > > [gluster config] > /etc/glusterfs/glusterd.vol > volume management > type mgmt/glusterd > option working-directory /var/lib/glusterd > option transport-type socket > option transport.socket.keepalive-time 10 > option transport.socket.keepalive-interval 2 > option transport.socket.read-fail-log off > option ping-timeout 30 > end-volume > > Best, > Yasu > _______________________________________________ > Gluster-users mailing list > Gluster-users at gluster.org > http://www.gluster.org/mailman/listinfo/gluster-users >-------------- next part -------------- An HTML attachment was scrubbed... URL: <http://www.gluster.org/pipermail/gluster-users/attachments/20150911/9d6a572d/attachment.html>