[Gluster-users] deleted files make bricks full ?

Tomoaki Sato
Mon, 01 Aug 2011 23:43:33 -0000

a simple way to reproduce the issue:
1) NFS mount and create 'foo' and umount.
2) NFS mount and delete 'foo' and umount.
3) replete 1) 2) till ENOSPC.

command logs are following:
[root at vhead-010 ~]# rpm -qa | grep gluster
glusterfs-fuse-3.1.5-1
glusterfs-core-3.1.5-1
[root at vhead-010 ~]# cat /etc/issue
CentOS release 5.6 (Final)
Kernel \r on an \m

[root at vhead-010 ~]# for i in small-{1..4}-4-private; do ssh $i df
/mnt/brick; do
ne
Filesystem           1K-blocks      Used Available Use% Mounted on
/dev/mapper/00002cde:00002cdf:8
                      103212320    192256 103020064   1% /mnt/brick
Filesystem           1K-blocks      Used Available Use% Mounted on
/dev/mapper/00002ceb:00002cec:8
                      103212320    192256 103020064   1% /mnt/brick
Filesystem           1K-blocks      Used Available Use% Mounted on
/dev/mapper/00002cf8:00002cf9:8
                      103212320    192256 103020064   1% /mnt/brick
Filesystem           1K-blocks      Used Available Use% Mounted on
/dev/mapper/00002d05:00002d06:8
                      103212320    192256 103020064   1% /mnt/brick
[root at vhead-010 ~]# mount small:/small /mnt
[root at vhead-010 ~]# ls /mnt
[root at vhead-010 ~]# dd if=/dev/zero of=/mnt/foo bs=1M count=1024
1024+0 records in
1024+0 records out
1073741824 bytes (1.1 GB) copied, 17.8419 seconds, 60.2 MB/s
[root at vhead-010 ~]# ls -l /mnt/foo
-rw-r--r-- 1 root root 1073741824 Aug  2 08:14 /mnt/foo
[root at vhead-010 ~]# umount /mnt
[root at vhead-010 ~]# for i in small-{1..4}-4-private; do ssh $i df
/mnt/brick; do
ne
Filesystem           1K-blocks      Used Available Use% Mounted on
/dev/mapper/00002cde:00002cdf:8
                      103212320   1241864 101970456   2% /mnt/brick
Filesystem           1K-blocks      Used Available Use% Mounted on
/dev/mapper/00002ceb:00002cec:8
                      103212320    192256 103020064   1% /mnt/brick
Filesystem           1K-blocks      Used Available Use% Mounted on
/dev/mapper/00002cf8:00002cf9:8
                      103212320    192256 103020064   1% /mnt/brick
Filesystem           1K-blocks      Used Available Use% Mounted on
/dev/mapper/00002d05:00002d06:8
                      103212320    192256 103020064   1% /mnt/brick
[root at vhead-010 ~]# mount small:/small /mnt
[root at vhead-010 ~]# rm -f /mnt/foo
[root at vhead-010 ~]# ls /mnt
[root at vhead-010 ~]# umount /mnt
[root at vhead-010 ~]# for i in small-{1..4}-4-private; do ssh $i df
/mnt/brick; do
ne
Filesystem           1K-blocks      Used Available Use% Mounted on
/dev/mapper/00002cde:00002cdf:8
                      103212320   1241864 101970456   2% /mnt/brick
Filesystem           1K-blocks      Used Available Use% Mounted on
/dev/mapper/00002ceb:00002cec:8
                      103212320    192256 103020064   1% /mnt/brick
Filesystem           1K-blocks      Used Available Use% Mounted on
/dev/mapper/00002cf8:00002cf9:8
                      103212320    192256 103020064   1% /mnt/brick
Filesystem           1K-blocks      Used Available Use% Mounted on
/dev/mapper/00002d05:00002d06:8
                      103212320    192256 103020064   1% /mnt/brick
[root at vhead-010 ~]# ssh small-1-4-private
[root at localhost ~]# du /mnt/brick
16      /mnt/brick/lost+found
24      /mnt/brick
[root at localhost ~]# ps ax | grep glusterfsd | grep -v grep
  7246 ?        Ssl    0:03 /opt/glusterfs/3.1.5/sbin/glusterfsd
--xlator-option
small-server.listen-port=24009 -s localhost --volfile-id
small.small-1-4-private
.mnt-brick -p
/etc/glusterd/vols/small/run/small-1-4-private-mnt-brick.pid --bri
ck-name /mnt/brick --brick-port 24009 -l
/var/log/glusterfs/bricks/mnt-brick.log

[root at localhost ~]# ls -l /proc/7246/fd
total 0
lrwx------ 1 root root 64 Aug  2 08:18 0 -> /dev/null
lrwx------ 1 root root 64 Aug  2 08:18 1 -> /dev/null
lrwx------ 1 root root 64 Aug  2 08:18 10 -> socket:[153304]
lrwx------ 1 root root 64 Aug  2 08:18 11 -> socket:[153306]
lrwx------ 1 root root 64 Aug  2 08:18 12 -> socket:[153388]
lrwx------ 1 root root 64 Aug  2 08:18 13 -> /mnt/brick/foo (deleted)
<===lrwx------ 1 root root 64 Aug  2 08:18 2 -> /dev/null
lr-x------ 1 root root 64 Aug  2 08:18 3 -> eventpoll:[153252]
l-wx------ 1 root root 64 Aug  2 08:18 4 ->
/var/log/glusterfs/bricks/mnt-brick.
log
lrwx------ 1 root root 64 Aug  2 08:18 5 ->
/etc/glusterd/vols/small/run/small-1
-4-private-mnt-brick.pid
lrwx------ 1 root root 64 Aug  2 08:18 6 -> socket:[153257]
lrwx------ 1 root root 64 Aug  2 08:18 7 -> socket:[153301]
lrwx------ 1 root root 64 Aug  2 08:18 8 -> /tmp/tmpfpuXk7N (deleted)
lrwx------ 1 root root 64 Aug  2 08:18 9 -> socket:[153297]
[root at localhost ~]# exit
[root at vhead-010 ~]# mount small:/small /mnt
[root at vhead-010 ~]# dd if=/dev/zero of=/mnt/foo bs=1M count=1024
1024+0 records in
1024+0 records out
1073741824 bytes (1.1 GB) copied, 21.4717 seconds, 50.0 MB/s
[root at vhead-010 ~]# ls -l /mnt/foo
-rw-r--r-- 1 root root 1073741824 Aug  2 08:19 /mnt/foo
[root at vhead-010 ~]# umount /mnt
[root at vhead-010 ~]# for i in small-{1..4}-4-private; do ssh $i df
/mnt/brick; do
ne
Filesystem           1K-blocks      Used Available Use% Mounted on
/dev/mapper/00002cde:00002cdf:8
                      103212320   2291472 100920848   3% /mnt/brick
Filesystem           1K-blocks      Used Available Use% Mounted on
/dev/mapper/00002ceb:00002cec:8
                      103212320    192256 103020064   1% /mnt/brick
Filesystem           1K-blocks      Used Available Use% Mounted on
/dev/mapper/00002cf8:00002cf9:8
                      103212320    192256 103020064   1% /mnt/brick
Filesystem           1K-blocks      Used Available Use% Mounted on
/dev/mapper/00002d05:00002d06:8
                      103212320    192256 103020064   1% /mnt/brick
[root at vhead-010 ~]# mount small:/small /mnt
[root at vhead-010 ~]# rm -f /mnt/foo
[root at vhead-010 ~]# ls /mnt
[root at vhead-010 ~]# umount /mnt
[root at vhead-010 ~]# for i in small-{1..4}-4-private; do ssh $i df
/mnt/brick; do
ne
Filesystem           1K-blocks      Used Available Use% Mounted on
/dev/mapper/00002cde:00002cdf:8
                      103212320   2291472 100920848   3% /mnt/brick
Filesystem           1K-blocks      Used Available Use% Mounted on
/dev/mapper/00002ceb:00002cec:8
                      103212320    192256 103020064   1% /mnt/brick
Filesystem           1K-blocks      Used Available Use% Mounted on
/dev/mapper/00002cf8:00002cf9:8
                      103212320    192256 103020064   1% /mnt/brick
Filesystem           1K-blocks      Used Available Use% Mounted on
/dev/mapper/00002d05:00002d06:8
                      103212320    192256 103020064   1% /mnt/brick
[root at vhead-010 ~]# ssh small-1-4-private ls -l /proc/7246/fd
total 0
lrwx------ 1 root root 64 Aug  2 08:18 0 -> /dev/null
lrwx------ 1 root root 64 Aug  2 08:18 1 -> /dev/null
lrwx------ 1 root root 64 Aug  2 08:18 10 -> socket:[153304]
lrwx------ 1 root root 64 Aug  2 08:18 11 -> socket:[153306]
lrwx------ 1 root root 64 Aug  2 08:18 12 -> socket:[153388]
lrwx------ 1 root root 64 Aug  2 08:18 13 -> /mnt/brick/foo (deleted)
<===lrwx------ 1 root root 64 Aug  2 08:21 14 -> /mnt/brick/foo (deleted)
<===lrwx------ 1 root root 64 Aug  2 08:18 2 -> /dev/null
lr-x------ 1 root root 64 Aug  2 08:18 3 -> eventpoll:[153252]
l-wx------ 1 root root 64 Aug  2 08:18 4 ->
/var/log/glusterfs/bricks/mnt-brick.
log
lrwx------ 1 root root 64 Aug  2 08:18 5 ->
/etc/glusterd/vols/small/run/small-1
-4-private-mnt-brick.pid
lrwx------ 1 root root 64 Aug  2 08:18 6 -> socket:[153257]
lrwx------ 1 root root 64 Aug  2 08:18 7 -> socket:[153301]
lrwx------ 1 root root 64 Aug  2 08:18 8 -> /tmp/tmpfpuXk7N (deleted)
lrwx------ 1 root root 64 Aug  2 08:18 9 -> socket:[153297]

Tomo Sato

(2011/08/02 7:14), Tomoaki Sato wrote:
> Hi,
>
> My simple test program, which repeat create-write-read-delete 64 of 1GB
> files on 100GB x 4 bricks from 4 NFS clients, fails due to ENOSPC.
> I found some glusterfsd have many file descriptors of same name, deleted
> files and these files fill the bricks.
> Is this know issue?
>
> Best,
>
> Tomo Sato
>
>
>
>
> _______________________________________________
> Gluster-users mailing list
> Gluster-users at gluster.org
> http://gluster.org/cgi-bin/mailman/listinfo/gluster-users