I had 2 nodes that I loaded almost 51TB to, then I expanded the cluster to 6 nodes, now I need to spread data out over them. Below I have the df -h from all 6 nodes, and you see that the original two nodes (clustr-04 clustr-05) have many mounts that are at 100% -- and even doing an ls -Rl of some of these directories are error-ing out, which is disconcerting after loading this much data. How can I level things out? I'm currently running the scale-n-defrag.sh on clustr-06 that has the client mounted on /mnt/glusterfs, but running this the other day from 04 didn't seem to help at all. Also, the run on clustr-06 is spitting out tons of "find: `setfattr': No such file or directory" What can I do to solve this? clustr-01 ======Filesystem Size Used Avail Use% Mounted on /dev/sdy3 133G 1.6G 124G 2% / tmpfs 4.0G 0 4.0G 0% /lib/init/rw udev 10M 336K 9.7M 4% /dev tmpfs 4.0G 0 4.0G 0% /dev/shm /dev/sdy1 7.4G 149M 6.9G 3% /var/log /dev/sdo1 1.4T 608G 754G 45% /mnt/data01 /dev/sdw1 1.4T 613G 749G 46% /mnt/data02 /dev/sdq1 1.4T 509G 853G 38% /mnt/data03 /dev/sdf1 1.4T 518G 845G 38% /mnt/data04 /dev/sdj1 1.4T 535G 827G 40% /mnt/data05 /dev/sdp1 1.4T 543G 819G 40% /mnt/data06 /dev/sds1 1.4T 536G 827G 40% /mnt/data07 /dev/sdk1 1.4T 531G 831G 39% /mnt/data08 /dev/sdg1 1.4T 511G 851G 38% /mnt/data09 /dev/sde1 1.4T 506G 856G 38% /mnt/data10 /dev/sdd1 1.4T 652G 710G 48% /mnt/data11 /dev/sdl1 1.4T 632G 730G 47% /mnt/data12 /dev/sdr1 1.4T 515G 848G 38% /mnt/data13 /dev/sdm1 1.4T 539G 824G 40% /mnt/data14 /dev/sdh1 1.4T 530G 832G 39% /mnt/data15 /dev/sda1 1.4T 525G 837G 39% /mnt/data16 /dev/sdi1 1.4T 515G 847G 38% /mnt/data17 /dev/sdu1 1.4T 517G 845G 38% /mnt/data18 /dev/sdc1 1.4T 518G 845G 38% /mnt/data19 /dev/sdv1 1.4T 517G 846G 38% /mnt/data20 /dev/sdt1 1.4T 548G 815G 41% /mnt/data21 /dev/sdb1 1.4T 559G 804G 42% /mnt/data22 /dev/sdn1 1.4T 615G 748G 46% /mnt/data23 /dev/sdx1 1.4T 618G 744G 46% /mnt/data24 clustr-02 ======Filesystem Size Used Avail Use% Mounted on /dev/sdy3 133G 1.2G 125G 1% / tmpfs 4.0G 0 4.0G 0% /lib/init/rw udev 10M 336K 9.7M 4% /dev tmpfs 4.0G 0 4.0G 0% /dev/shm /dev/sdy1 7.4G 151M 6.9G 3% /var/log /dev/sdh1 1.4T 551G 812G 41% /mnt/data01 /dev/sds1 1.4T 566G 797G 42% /mnt/data02 /dev/sdc1 1.4T 533G 829G 40% /mnt/data03 /dev/sdx1 1.4T 549G 813G 41% /mnt/data04 /dev/sdu1 1.4T 519G 844G 39% /mnt/data05 /dev/sdf1 1.4T 539G 823G 40% /mnt/data06 /dev/sdt1 1.4T 559G 803G 42% /mnt/data07 /dev/sdj1 1.4T 530G 832G 39% /mnt/data08 /dev/sdk1 1.4T 549G 814G 41% /mnt/data09 /dev/sdd1 1.4T 544G 818G 40% /mnt/data10 /dev/sdr1 1.4T 559G 803G 42% /mnt/data11 /dev/sdi1 1.4T 529G 833G 39% /mnt/data12 /dev/sdb1 1.4T 525G 837G 39% /mnt/data13 /dev/sdq1 1.4T 522G 840G 39% /mnt/data14 /dev/sde1 1.4T 557G 805G 41% /mnt/data15 /dev/sdn1 1.4T 576G 787G 43% /mnt/data16 /dev/sdw1 1.4T 514G 849G 38% /mnt/data17 /dev/sdl1 1.4T 519G 843G 39% /mnt/data18 /dev/sdp1 1.4T 541G 821G 40% /mnt/data19 /dev/sdg1 1.4T 552G 810G 41% /mnt/data20 /dev/sdo1 1.4T 509G 853G 38% /mnt/data21 /dev/sda1 1.4T 510G 853G 38% /mnt/data22 /dev/sdm1 1.4T 503G 860G 37% /mnt/data23 /dev/sdv1 1.4T 515G 847G 38% /mnt/data24 clustr-03 ======Filesystem Size Used Avail Use% Mounted on /dev/sdy3 133G 1.5G 124G 2% / tmpfs 4.0G 0 4.0G 0% /lib/init/rw udev 10M 352K 9.7M 4% /dev tmpfs 4.0G 0 4.0G 0% /dev/shm /dev/sdy1 7.4G 1.1G 6.0G 16% /var/log /dev/sdn1 1.4T 558G 804G 41% /mnt/data01 /dev/sdi1 1.4T 556G 806G 41% /mnt/data02 /dev/sdm1 1.4T 511G 851G 38% /mnt/data03 /dev/sdj1 1.4T 487G 875G 36% /mnt/data04 /dev/sdl1 1.4T 523G 839G 39% /mnt/data05 /dev/sdr1 1.4T 542G 820G 40% /mnt/data06 /dev/sdu1 1.4T 472G 891G 35% /mnt/data07 /dev/sdq1 1.4T 464G 898G 35% /mnt/data08 /dev/sdh1 1.4T 501G 861G 37% /mnt/data09 /dev/sdg1 1.4T 489G 874G 36% /mnt/data10 /dev/sde1 1.4T 452G 910G 34% /mnt/data11 /dev/sdv1 1.4T 462G 900G 34% /mnt/data12 /dev/sdc1 1.4T 460G 903G 34% /mnt/data13 /dev/sdf1 1.4T 467G 896G 35% /mnt/data14 /dev/sdt1 1.4T 495G 867G 37% /mnt/data15 /dev/sdk1 1.4T 491G 871G 37% /mnt/data16 /dev/sds1 1.4T 378G 985G 28% /mnt/data18 /dev/sda1 1.4T 492G 870G 37% /mnt/data19 /dev/sdx1 1.4T 498G 864G 37% /mnt/data20 /dev/sdw1 1.4T 455G 908G 34% /mnt/data21 /dev/sdo1 1.4T 447G 915G 33% /mnt/data22 /dev/sdb1 1.4T 472G 891G 35% /mnt/data23 /dev/sdd1 1.4T 482G 880G 36% /mnt/data24 /dev/sdp1 1.4T 424G 939G 32% /mnt/data17 clustr-04 ======Filesystem Size Used Avail Use% Mounted on /dev/sdy4 141G 8.4G 126G 7% / tmpfs 4.0G 0 4.0G 0% /lib/init/rw udev 10M 344K 9.7M 4% /dev tmpfs 4.0G 0 4.0G 0% /dev/shm /dev/sdy1 228M 49M 168M 23% /boot /dev/sdy2 3.7G 82M 3.4G 3% /var/log /dev/sdw1 1.4T 1.4T 0 100% /mnt/data01 /dev/sdp1 1.4T 1.4T 0 100% /mnt/data02 /dev/sdm1 1.4T 1.4T 5.3G 100% /mnt/data03 /dev/sde1 1.4T 1.4T 5.3G 100% /mnt/data04 /dev/sdh1 1.4T 1.4T 2.1G 100% /mnt/data05 /dev/sdt1 1.4T 1.4T 2.1G 100% /mnt/data06 /dev/sdg1 1.4T 1.4T 0 100% /mnt/data07 /dev/sdk1 1.4T 1.4T 0 100% /mnt/data08 /dev/sda1 1.4T 1.4T 0 100% /mnt/data09 /dev/sdn1 1.4T 1.4T 0 100% /mnt/data10 /dev/sdc1 1.4T 1.4T 4.3G 100% /mnt/data11 /dev/sdd1 1.4T 1.4T 4.3G 100% /mnt/data12 /dev/sdf1 1.4T 1.4T 9.3G 100% /mnt/data13 /dev/sdx1 1.4T 1.4T 9.3G 100% /mnt/data14 /dev/sdi1 1.4T 1.4T 2.6G 100% /mnt/data15 /dev/sdj1 1.4T 1.4T 2.6G 100% /mnt/data16 /dev/sdl1 1.4T 1.4T 0 100% /mnt/data17 /dev/sdq1 1.4T 1.4T 0 100% /mnt/data18 /dev/sdv1 1.4T 1.4T 6.2G 100% /mnt/data19 /dev/sdo1 1.4T 1.4T 6.2G 100% /mnt/data20 /dev/sds1 1.4T 1.4T 7.9G 100% /mnt/data21 /dev/sdb1 1.4T 1.4T 7.9G 100% /mnt/data22 /dev/sdr1 1.4T 1.4T 0 100% /mnt/data23 /dev/sdu1 1.4T 1.4T 0 100% /mnt/data24 /etc/glusterfs/glusterfs.vol 97T 51T 46T 53% /mnt/glusterfs clustr-05 ======Filesystem Size Used Avail Use% Mounted on /dev/sdy4 141G 1.5G 133G 2% / tmpfs 4.0G 0 4.0G 0% /lib/init/rw udev 10M 356K 9.7M 4% /dev tmpfs 4.0G 0 4.0G 0% /dev/shm /dev/sdy1 236M 39M 185M 18% /boot /dev/sdy3 3.7G 80M 3.5G 3% /var/log /dev/sdl1 1.4T 1.4T 6.1G 100% /mnt/data01 /dev/sdq1 1.4T 1.4T 6.1G 100% /mnt/data02 /dev/sdp1 1.4T 1.4T 0 100% /mnt/data03 /dev/sdt1 1.4T 1.4T 0 100% /mnt/data04 /dev/sdi1 1.4T 1.4T 0 100% /mnt/data05 /dev/sdf1 1.4T 1.4T 0 100% /mnt/data06 /dev/sdc1 1.4T 1.4T 0 100% /mnt/data07 /dev/sdv1 1.4T 1.4T 0 100% /mnt/data08 /dev/sdm1 1.4T 1.4T 0 100% /mnt/data09 /dev/sdr1 1.4T 1.4T 0 100% /mnt/data10 /dev/sdg1 1.4T 1.4T 15G 99% /mnt/data11 /dev/sdu1 1.4T 1.4T 15G 99% /mnt/data12 /dev/sdh1 1.4T 1.4T 0 100% /mnt/data13 /dev/sda1 1.4T 1.4T 0 100% /mnt/data14 /dev/sdb1 1.4T 1.4T 0 100% /mnt/data15 /dev/sdd1 1.4T 1.4T 0 100% /mnt/data16 /dev/sdn1 1.4T 1.4T 0 100% /mnt/data17 /dev/sdk1 1.4T 1.4T 0 100% /mnt/data18 /dev/sdj1 1.4T 1.4T 824M 100% /mnt/data19 /dev/sde1 1.4T 1.4T 824M 100% /mnt/data20 /dev/sdw1 1.4T 1.4T 0 100% /mnt/data21 /dev/sdx1 1.4T 1.4T 0 100% /mnt/data22 /dev/sds1 1.4T 1.4T 0 100% /mnt/data23 /dev/sdo1 1.4T 1.4T 0 100% /mnt/data24 clustr06 ======Filesystem Size Used Avail Use% Mounted on /dev/sdy3 138G 1.2G 130G 1% / tmpfs 4.0G 0 4.0G 0% /lib/init/rw udev 10M 344K 9.7M 4% /dev tmpfs 4.0G 0 4.0G 0% /dev/shm /dev/sdy1 7.4G 149M 6.9G 3% /var/log /dev/sdb1 1.4T 483M 1.4T 1% /mnt/data01 /dev/sdi1 1.4T 483M 1.4T 1% /mnt/data02 /dev/sdj1 1.4T 286M 1.4T 1% /mnt/data03 /dev/sdp1 1.4T 286M 1.4T 1% /mnt/data04 /dev/sdr1 1.4T 286M 1.4T 1% /mnt/data05 /dev/sdx1 1.4T 286M 1.4T 1% /mnt/data06 /dev/sdv1 1.4T 286M 1.4T 1% /mnt/data07 /dev/sdt1 1.4T 286M 1.4T 1% /mnt/data08 /dev/sdl1 1.4T 286M 1.4T 1% /mnt/data09 /dev/sdg1 1.4T 286M 1.4T 1% /mnt/data10 /dev/sde1 1.4T 286M 1.4T 1% /mnt/data11 /dev/sdn1 1.4T 286M 1.4T 1% /mnt/data12 /dev/sda1 1.4T 286M 1.4T 1% /mnt/data13 /dev/sdc1 1.4T 286M 1.4T 1% /mnt/data14 /dev/sdk1 1.4T 285M 1.4T 1% /mnt/data15 /dev/sdw1 1.4T 286M 1.4T 1% /mnt/data16 /dev/sdq1 1.4T 286M 1.4T 1% /mnt/data17 /dev/sdu1 1.4T 286M 1.4T 1% /mnt/data18 /dev/sdd1 1.4T 286M 1.4T 1% /mnt/data19 /dev/sdh1 1.4T 286M 1.4T 1% /mnt/data20 /dev/sdm1 1.4T 285M 1.4T 1% /mnt/data21 /dev/sdo1 1.4T 285M 1.4T 1% /mnt/data22 /dev/sds1 1.4T 286M 1.4T 1% /mnt/data23 /dev/sdf1 1.4T 286M 1.4T 1% /mnt/data24 /etc/glusterfs/glusterfs.vol 97T 51T 46T 53% /mnt/glusterfs -- http://philcryer.com