[root at ml-ctl-ser01 sunyuyusun]$ salt "ml-storage-ser2*.nmg01"
cmd.run
"getfattr -d -m. -e hex
/data2/bricks/project2/371_37829/face_landmarks"
ml-storage-ser24.nmg01:
getfattr: Removing leading '/' from absolute path names
# file: data2/bricks/project2/371_37829/face_landmarks
trusted.gfid=0x3764e261741d423d8c8eb6ad5b716429
trusted.glusterfs.dht=0xdbde0bb100000000b0c30c2eb3cf3cf0
trusted.glusterfs.quota.dirty=0x3000
trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1=0x00000000000d760000000000000000010000000000000003
trusted.glusterfs.quota.size.1=0x00000000000d760000000000000000010000000000000003
ml-storage-ser26.nmg01:
getfattr: Removing leading '/' from absolute path names
# file: data2/bricks/project2/371_37829/face_landmarks
trusted.gfid=0x3764e261741d423d8c8eb6ad5b716429
trusted.glusterfs.dht=0xdbde0bb100000000d249248fd5555551
trusted.glusterfs.quota.dirty=0x3000
trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1=0x000000000000000000000000000000000000000000000003
trusted.glusterfs.quota.size.1=0x000000000000000000000000000000000000000000000003
ml-storage-ser23.nmg01:
getfattr: Removing leading '/' from absolute path names
# file: data2/bricks/project2/371_37829/face_landmarks
trusted.gfid=0x3764e261741d423d8c8eb6ad5b716429
trusted.glusterfs.dht=0xdbde0bb1000000008f3cf3cd9249248f
trusted.glusterfs.quota.dirty=0x3000
trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1=0x000000000000000000000000000000000000000000000003
trusted.glusterfs.quota.size.1=0x000000000000000000000000000000000000000000000003
ml-storage-ser27.nmg01:
getfattr: Removing leading '/' from absolute path names
# file: data2/bricks/project2/371_37829/face_landmarks
trusted.gfid=0x3764e261741d423d8c8eb6ad5b716429
trusted.glusterfs.dht=0xdbde0bb100000000d249248fd5555551
trusted.glusterfs.quota.dirty=0x3000
trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1=0x000000000000000000000000000000000000000000000003
trusted.glusterfs.quota.size.1=0x000000000000000000000000000000000000000000000003
ml-storage-ser25.nmg01:
getfattr: Removing leading '/' from absolute path names
# file: data2/bricks/project2/371_37829/face_landmarks
trusted.gfid=0x3764e261741d423d8c8eb6ad5b716429
trusted.glusterfs.dht=0xdbde0bb100000000b0c30c2eb3cf3cf0
trusted.glusterfs.quota.dirty=0x3000
trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1=0x00000000000d760000000000000000010000000000000003
trusted.glusterfs.quota.size.1=0x00000000000d760000000000000000010000000000000003
ml-storage-ser28.nmg01:
getfattr: Removing leading '/' from absolute path names
# file: data2/bricks/project2/371_37829/face_landmarks
trusted.gfid=0x3764e261741d423d8c8eb6ad5b716429
trusted.glusterfs.dht=0xdbde0bb100000000f3cf3cf0f6db6db2
trusted.glusterfs.quota.dirty=0x3000
trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1=0x000000000000000000000000000000000000000000000003
trusted.glusterfs.quota.size.1=0x000000000000000000000000000000000000000000000003
ml-storage-ser29.nmg01:
getfattr: Removing leading '/' from absolute path names
# file: data2/bricks/project2/371_37829/face_landmarks
trusted.gfid=0x3764e261741d423d8c8eb6ad5b716429
trusted.glusterfs.dht=0xdbde0bb100000000f3cf3cf0f6db6db2
trusted.glusterfs.quota.dirty=0x3000
trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1=0x000000000000000000000000000000000000000000000003
trusted.glusterfs.quota.size.1=0x000000000000000000000000000000000000000000000003
ml-storage-ser20.nmg01:
getfattr: Removing leading '/' from absolute path names
# file: data2/bricks/project2/371_37829/face_landmarks
trusted.gfid=0x3764e261741d423d8c8eb6ad5b716429
trusted.glusterfs.dht=0xdbde0bb1000000008c30c30a8f3cf3cc
trusted.glusterfs.quota.dirty=0x3000
trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1=0x000000000000000000000000000000000000000000000003
trusted.glusterfs.quota.size.1=0x000000000000000000000000000000000000000000000003
ml-storage-ser22.nmg01:
getfattr: Removing leading '/' from absolute path names
# file: data2/bricks/project2/371_37829/face_landmarks
trusted.gfid=0x3764e261741d423d8c8eb6ad5b716429
trusted.glusterfs.dht=0xdbde0bb1000000008f3cf3cd9249248f
trusted.glusterfs.quota.dirty=0x3000
trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1=0x000000000000000000000000000000000000000000000003
trusted.glusterfs.quota.size.1=0x000000000000000000000000000000000000000000000003
ml-storage-ser21.nmg01:
getfattr: Removing leading '/' from absolute path names
# file: data2/bricks/project2/371_37829/face_landmarks
trusted.gfid=0x3764e261741d423d8c8eb6ad5b716429
trusted.glusterfs.dht=0xdbde0bb1000000008c30c30a8f3cf3cc
trusted.glusterfs.quota.dirty=0x3000
trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1=0x000000000000000000000000000000000000000000000003
trusted.glusterfs.quota.size.1=0x000000000000000000000000000000000000000000000003
[root at ml-ctl-ser01 sunyuyusun]$ salt "ml-storage-ser2*.nmg01"
cmd.run
"getfattr -d -m. -e hex
/data3/bricks/project2/371_37829/face_landmarks"
ml-storage-ser28.nmg01:
getfattr: Removing leading '/' from absolute path names
# file: data3/bricks/project2/371_37829/face_landmarks
trusted.gfid=0x3764e261741d423d8c8eb6ad5b716429
trusted.glusterfs.dht=0xdbde0bb1000000009555555398618615
trusted.glusterfs.quota.dirty=0x3000
trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1=0x00000000001aec0000000000000000020000000000000003
trusted.glusterfs.quota.size.1=0x00000000001aec0000000000000000020000000000000003
ml-storage-ser25.nmg01:
getfattr: Removing leading '/' from absolute path names
# file: data3/bricks/project2/371_37829/face_landmarks
trusted.gfid=0x3764e261741d423d8c8eb6ad5b716429
trusted.glusterfs.dht=0xdbde0bb100000000892492478c30c309
trusted.glusterfs.quota.dirty=0x3000
trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1=0x000000000000000000000000000000000000000000000003
trusted.glusterfs.quota.size.1=0x000000000000000000000000000000000000000000000003
ml-storage-ser24.nmg01:
getfattr: Removing leading '/' from absolute path names
# file: data3/bricks/project2/371_37829/face_landmarks
trusted.gfid=0x3764e261741d423d8c8eb6ad5b716429
trusted.glusterfs.dht=0xdbde0bb100000000892492478c30c309
trusted.glusterfs.quota.dirty=0x3000
trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1=0x000000000000000000000000000000000000000000000003
trusted.glusterfs.quota.size.1=0x000000000000000000000000000000000000000000000003
ml-storage-ser27.nmg01:
getfattr: Removing leading '/' from absolute path names
# file: data3/bricks/project2/371_37829/face_landmarks
trusted.gfid=0x3764e261741d423d8c8eb6ad5b716429
trusted.glusterfs.dht=0xdbde0bb1000000009249249095555552
trusted.glusterfs.quota.dirty=0x3000
trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1=0x000000000000000000000000000000000000000000000003
trusted.glusterfs.quota.size.1=0x000000000000000000000000000000000000000000000003
ml-storage-ser23.nmg01:
getfattr: Removing leading '/' from absolute path names
# file: data3/bricks/project2/371_37829/face_landmarks
trusted.gfid=0x3764e261741d423d8c8eb6ad5b716429
trusted.glusterfs.dht=0xdbde0bb10000000079e79e787cf3cf3a
trusted.glusterfs.quota.dirty=0x3000
trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1=0x000000000000000000000000000000000000000000000003
trusted.glusterfs.quota.size.1=0x000000000000000000000000000000000000000000000003
ml-storage-ser22.nmg01:
getfattr: Removing leading '/' from absolute path names
# file: data3/bricks/project2/371_37829/face_landmarks
trusted.gfid=0x3764e261741d423d8c8eb6ad5b716429
trusted.glusterfs.dht=0xdbde0bb10000000079e79e787cf3cf3a
trusted.glusterfs.quota.dirty=0x3000
trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1=0x000000000000000000000000000000000000000000000003
trusted.glusterfs.quota.size.1=0x000000000000000000000000000000000000000000000003
ml-storage-ser29.nmg01:
getfattr: Removing leading '/' from absolute path names
# file: data3/bricks/project2/371_37829/face_landmarks
trusted.gfid=0x3764e261741d423d8c8eb6ad5b716429
trusted.glusterfs.dht=0xdbde0bb1000000009555555398618615
trusted.glusterfs.quota.dirty=0x3000
trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1=0x00000000001aec0000000000000000020000000000000003
trusted.glusterfs.quota.size.1=0x00000000001aec0000000000000000020000000000000003
ml-storage-ser26.nmg01:
getfattr: Removing leading '/' from absolute path names
# file: data3/bricks/project2/371_37829/face_landmarks
trusted.gfid=0x3764e261741d423d8c8eb6ad5b716429
trusted.glusterfs.dht=0xdbde0bb1000000009249249095555552
trusted.glusterfs.quota.dirty=0x3000
trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1=0x000000000000000000000000000000000000000000000003
trusted.glusterfs.quota.size.1=0x000000000000000000000000000000000000000000000003
ml-storage-ser21.nmg01:
getfattr: Removing leading '/' from absolute path names
# file: data3/bricks/project2/371_37829/face_landmarks
trusted.gfid=0x3764e261741d423d8c8eb6ad5b716429
trusted.glusterfs.dht=0xdbde0bb100000000586186175b6db6d9
trusted.glusterfs.quota.dirty=0x3000
trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1=0x000000000000000000000000000000000000000000000003
trusted.glusterfs.quota.size.1=0x000000000000000000000000000000000000000000000003
ml-storage-ser20.nmg01:
getfattr: Removing leading '/' from absolute path names
# file: data3/bricks/project2/371_37829/face_landmarks
trusted.gfid=0x3764e261741d423d8c8eb6ad5b716429
trusted.glusterfs.dht=0xdbde0bb100000000586186175b6db6d9
trusted.glusterfs.quota.dirty=0x3000
trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1=0x000000000000000000000000000000000000000000000003
trusted.glusterfs.quota.size.1=0x000000000000000000000000000000000000000000000003
[root at ml-ctl-ser01 sunyuyusun]$ salt "ml-storage-ser2*.nmg01"
cmd.run
"getfattr -d -m. -e hex
/data4/bricks/project2/371_37829/face_landmarks"
ml-storage-ser25.nmg01:
getfattr: Removing leading '/' from absolute path names
# file: data4/bricks/project2/371_37829/face_landmarks
trusted.gfid=0x3764e261741d423d8c8eb6ad5b716429
trusted.glusterfs.dht=0xdbde0bb100000000a4924922a79e79e4
trusted.glusterfs.quota.dirty=0x3000
trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1=0x000000000000000000000000000000000000000000000003
trusted.glusterfs.quota.size.1=0x000000000000000000000000000000000000000000000003
ml-storage-ser23.nmg01:
getfattr: Removing leading '/' from absolute path names
# file: data4/bricks/project2/371_37829/face_landmarks
trusted.gfid=0x3764e261741d423d8c8eb6ad5b716429
trusted.glusterfs.dht=0xdbde0bb100000000a186185fa4924921
trusted.glusterfs.quota.dirty=0x3000
trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1=0x000000000000000000000000000000000000000000000003
trusted.glusterfs.quota.size.1=0x000000000000000000000000000000000000000000000003
ml-storage-ser24.nmg01:
getfattr: Removing leading '/' from absolute path names
# file: data4/bricks/project2/371_37829/face_landmarks
trusted.gfid=0x3764e261741d423d8c8eb6ad5b716429
trusted.glusterfs.dht=0xdbde0bb100000000a4924922a79e79e4
trusted.glusterfs.quota.dirty=0x3000
trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1=0x000000000000000000000000000000000000000000000003
trusted.glusterfs.quota.size.1=0x000000000000000000000000000000000000000000000003
ml-storage-ser28.nmg01:
getfattr: Removing leading '/' from absolute path names
# file: data4/bricks/project2/371_37829/face_landmarks
trusted.gfid=0x3764e261741d423d8c8eb6ad5b716429
trusted.glusterfs.dht=0xdbde0bb100000000aaaaaaa8adb6db6a
trusted.glusterfs.quota.dirty=0x3000
trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1=0x000000000000000000000000000000000000000000000003
trusted.glusterfs.quota.size.1=0x000000000000000000000000000000000000000000000003
ml-storage-ser27.nmg01:
getfattr: Removing leading '/' from absolute path names
# file: data4/bricks/project2/371_37829/face_landmarks
trusted.gfid=0x3764e261741d423d8c8eb6ad5b716429
trusted.glusterfs.dht=0xdbde0bb100000000a79e79e5aaaaaaa7
trusted.glusterfs.quota.dirty=0x3000
trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1=0x000000000000000000000000000000000000000000000003
trusted.glusterfs.quota.size.1=0x000000000000000000000000000000000000000000000003
ml-storage-ser26.nmg01:
getfattr: Removing leading '/' from absolute path names
# file: data4/bricks/project2/371_37829/face_landmarks
trusted.gfid=0x3764e261741d423d8c8eb6ad5b716429
trusted.glusterfs.dht=0xdbde0bb100000000a79e79e5aaaaaaa7
trusted.glusterfs.quota.dirty=0x3000
trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1=0x000000000000000000000000000000000000000000000003
trusted.glusterfs.quota.size.1=0x000000000000000000000000000000000000000000000003
ml-storage-ser21.nmg01:
getfattr: Removing leading '/' from absolute path names
# file: data4/bricks/project2/371_37829/face_landmarks
trusted.gfid=0x3764e261741d423d8c8eb6ad5b716429
trusted.glusterfs.dht=0xdbde0bb1000000009e79e79ca186185e
trusted.glusterfs.quota.dirty=0x3000
trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1=0x000000000000000000000000000000000000000000000003
trusted.glusterfs.quota.size.1=0x000000000000000000000000000000000000000000000003
ml-storage-ser22.nmg01:
getfattr: Removing leading '/' from absolute path names
# file: data4/bricks/project2/371_37829/face_landmarks
trusted.gfid=0x3764e261741d423d8c8eb6ad5b716429
trusted.glusterfs.dht=0xdbde0bb100000000a186185fa4924921
trusted.glusterfs.quota.dirty=0x3000
trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1=0x000000000000000000000000000000000000000000000003
trusted.glusterfs.quota.size.1=0x000000000000000000000000000000000000000000000003
ml-storage-ser29.nmg01:
getfattr: Removing leading '/' from absolute path names
# file: data4/bricks/project2/371_37829/face_landmarks
trusted.gfid=0x3764e261741d423d8c8eb6ad5b716429
trusted.glusterfs.dht=0xdbde0bb100000000aaaaaaa8adb6db6a
trusted.glusterfs.quota.dirty=0x3000
trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1=0x000000000000000000000000000000000000000000000003
trusted.glusterfs.quota.size.1=0x000000000000000000000000000000000000000000000003
ml-storage-ser20.nmg01:
getfattr: Removing leading '/' from absolute path names
# file: data4/bricks/project2/371_37829/face_landmarks
trusted.gfid=0x3764e261741d423d8c8eb6ad5b716429
trusted.glusterfs.dht=0xdbde0bb1000000009e79e79ca186185e
trusted.glusterfs.quota.dirty=0x3000
trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1=0x000000000000000000000000000000000000000000000003
trusted.glusterfs.quota.size.1=0x000000000000000000000000000000000000000000000003
[root at ml-ctl-ser01 sunyuyusun]$ salt "ml-storage-ser2*.nmg01"
cmd.run
"getfattr -d -m. -e hex /data2/bricks/project2/371_37829/test-dir"
ml-storage-ser23.nmg01:
getfattr: Removing leading '/' from absolute path names
# file: data2/bricks/project2/371_37829/test-dir
trusted.gfid=0x57e0a8945a224ab4be2c6c71eada6217
trusted.glusterfs.dht=0xdbde0bb1000000000f3cf3cf12492491
trusted.glusterfs.quota.dirty=0x3000
trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1=0x000000000000000000000000000000000000000000000001
trusted.glusterfs.quota.size.1=0x000000000000000000000000000000000000000000000001
ml-storage-ser24.nmg01:
getfattr: Removing leading '/' from absolute path names
# file: data2/bricks/project2/371_37829/test-dir
trusted.gfid=0x57e0a8945a224ab4be2c6c71eada6217
trusted.glusterfs.dht=0xdbde0bb10000000030c30c3033cf3cf2
trusted.glusterfs.quota.dirty=0x3000
trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1=0x000000000000000000000000000000000000000000000001
trusted.glusterfs.quota.size.1=0x000000000000000000000000000000000000000000000001
ml-storage-ser28.nmg01:
getfattr: Removing leading '/' from absolute path names
# file: data2/bricks/project2/371_37829/test-dir
trusted.gfid=0x57e0a8945a224ab4be2c6c71eada6217
trusted.glusterfs.dht=0xdbde0bb10000000073cf3cf276db6db4
trusted.glusterfs.quota.dirty=0x3000
trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1=0x000000000000000000000000000000000000000000000001
trusted.glusterfs.quota.size.1=0x000000000000000000000000000000000000000000000001
ml-storage-ser25.nmg01:
getfattr: Removing leading '/' from absolute path names
# file: data2/bricks/project2/371_37829/test-dir
trusted.gfid=0x57e0a8945a224ab4be2c6c71eada6217
trusted.glusterfs.dht=0xdbde0bb10000000030c30c3033cf3cf2
trusted.glusterfs.quota.dirty=0x3000
trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1=0x000000000000000000000000000000000000000000000001
trusted.glusterfs.quota.size.1=0x000000000000000000000000000000000000000000000001
ml-storage-ser26.nmg01:
getfattr: Removing leading '/' from absolute path names
# file: data2/bricks/project2/371_37829/test-dir
trusted.gfid=0x57e0a8945a224ab4be2c6c71eada6217
trusted.glusterfs.dht=0xdbde0bb1000000005249249155555553
trusted.glusterfs.quota.dirty=0x3000
trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1=0x000000000000000000000000000000000000000000000001
trusted.glusterfs.quota.size.1=0x000000000000000000000000000000000000000000000001
ml-storage-ser27.nmg01:
getfattr: Removing leading '/' from absolute path names
# file: data2/bricks/project2/371_37829/test-dir
trusted.gfid=0x57e0a8945a224ab4be2c6c71eada6217
trusted.glusterfs.dht=0xdbde0bb1000000005249249155555553
trusted.glusterfs.quota.dirty=0x3000
trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1=0x000000000000000000000000000000000000000000000001
trusted.glusterfs.quota.size.1=0x000000000000000000000000000000000000000000000001
ml-storage-ser22.nmg01:
getfattr: Removing leading '/' from absolute path names
# file: data2/bricks/project2/371_37829/test-dir
trusted.gfid=0x57e0a8945a224ab4be2c6c71eada6217
trusted.glusterfs.dht=0xdbde0bb1000000000f3cf3cf12492491
trusted.glusterfs.quota.dirty=0x3000
trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1=0x000000000000000000000000000000000000000000000001
trusted.glusterfs.quota.size.1=0x000000000000000000000000000000000000000000000001
ml-storage-ser29.nmg01:
getfattr: Removing leading '/' from absolute path names
# file: data2/bricks/project2/371_37829/test-dir
trusted.gfid=0x57e0a8945a224ab4be2c6c71eada6217
trusted.glusterfs.dht=0xdbde0bb10000000073cf3cf276db6db4
trusted.glusterfs.quota.dirty=0x3000
trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1=0x000000000000000000000000000000000000000000000001
trusted.glusterfs.quota.size.1=0x000000000000000000000000000000000000000000000001
ml-storage-ser21.nmg01:
getfattr: Removing leading '/' from absolute path names
# file: data2/bricks/project2/371_37829/test-dir
trusted.gfid=0x57e0a8945a224ab4be2c6c71eada6217
trusted.glusterfs.dht=0xdbde0bb1000000000c30c30c0f3cf3ce
trusted.glusterfs.quota.dirty=0x3000
trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1=0x000000000000000000000000000000000000000000000001
trusted.glusterfs.quota.size.1=0x000000000000000000000000000000000000000000000001
ml-storage-ser20.nmg01:
getfattr: Removing leading '/' from absolute path names
# file: data2/bricks/project2/371_37829/test-dir
trusted.gfid=0x57e0a8945a224ab4be2c6c71eada6217
trusted.glusterfs.dht=0xdbde0bb1000000000c30c30c0f3cf3ce
trusted.glusterfs.quota.dirty=0x3000
trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1=0x000000000000000000000000000000000000000000000001
trusted.glusterfs.quota.size.1=0x000000000000000000000000000000000000000000000001
[root at ml-ctl-ser01 sunyuyusun]$ salt "ml-storage-ser2*.nmg01"
cmd.run
"getfattr -d -m. -e hex /data3/bricks/project2/371_37829/test-dir"
ml-storage-ser23.nmg01:
getfattr: Removing leading '/' from absolute path names
# file: data3/bricks/project2/371_37829/test-dir
trusted.gfid=0x57e0a8945a224ab4be2c6c71eada6217
trusted.glusterfs.dht=0xdbde0bb100000000f9e79e76fcf3cf38
trusted.glusterfs.quota.dirty=0x3000
trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1=0x000000000000000000000000000000000000000000000001
trusted.glusterfs.quota.size.1=0x000000000000000000000000000000000000000000000001
ml-storage-ser26.nmg01:
getfattr: Removing leading '/' from absolute path names
# file: data3/bricks/project2/371_37829/test-dir
trusted.gfid=0x57e0a8945a224ab4be2c6c71eada6217
trusted.glusterfs.dht=0xdbde0bb1000000001249249215555554
trusted.glusterfs.quota.dirty=0x3000
trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1=0x000000000000000000000000000000000000000000000001
trusted.glusterfs.quota.size.1=0x000000000000000000000000000000000000000000000001
ml-storage-ser25.nmg01:
getfattr: Removing leading '/' from absolute path names
# file: data3/bricks/project2/371_37829/test-dir
trusted.gfid=0x57e0a8945a224ab4be2c6c71eada6217
trusted.glusterfs.dht=0xdbde0bb100000000092492490c30c30b
trusted.glusterfs.quota.dirty=0x3000
trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1=0x000000000000000000000000000000000000000000000001
trusted.glusterfs.quota.size.1=0x000000000000000000000000000000000000000000000001
ml-storage-ser28.nmg01:
getfattr: Removing leading '/' from absolute path names
# file: data3/bricks/project2/371_37829/test-dir
trusted.gfid=0x57e0a8945a224ab4be2c6c71eada6217
trusted.glusterfs.dht=0xdbde0bb1000000001555555518618617
trusted.glusterfs.quota.dirty=0x3000
trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1=0x000000000000000000000000000000000000000000000001
trusted.glusterfs.quota.size.1=0x000000000000000000000000000000000000000000000001
ml-storage-ser24.nmg01:
getfattr: Removing leading '/' from absolute path names
# file: data3/bricks/project2/371_37829/test-dir
trusted.gfid=0x57e0a8945a224ab4be2c6c71eada6217
trusted.glusterfs.dht=0xdbde0bb100000000092492490c30c30b
trusted.glusterfs.quota.dirty=0x3000
trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1=0x000000000000000000000000000000000000000000000001
trusted.glusterfs.quota.size.1=0x000000000000000000000000000000000000000000000001
ml-storage-ser22.nmg01:
getfattr: Removing leading '/' from absolute path names
# file: data3/bricks/project2/371_37829/test-dir
trusted.gfid=0x57e0a8945a224ab4be2c6c71eada6217
trusted.glusterfs.dht=0xdbde0bb100000000f9e79e76fcf3cf38
trusted.glusterfs.quota.dirty=0x3000
trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1=0x000000000000000000000000000000000000000000000001
trusted.glusterfs.quota.size.1=0x000000000000000000000000000000000000000000000001
ml-storage-ser29.nmg01:
getfattr: Removing leading '/' from absolute path names
# file: data3/bricks/project2/371_37829/test-dir
trusted.gfid=0x57e0a8945a224ab4be2c6c71eada6217
trusted.glusterfs.dht=0xdbde0bb1000000001555555518618617
trusted.glusterfs.quota.dirty=0x3000
trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1=0x000000000000000000000000000000000000000000000001
trusted.glusterfs.quota.size.1=0x000000000000000000000000000000000000000000000001
ml-storage-ser21.nmg01:
getfattr: Removing leading '/' from absolute path names
# file: data3/bricks/project2/371_37829/test-dir
trusted.gfid=0x57e0a8945a224ab4be2c6c71eada6217
trusted.glusterfs.dht=0xdbde0bb100000000d8618615db6db6d7
trusted.glusterfs.quota.dirty=0x3000
trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1=0x000000000000000000000000000000000000000000000001
trusted.glusterfs.quota.size.1=0x000000000000000000000000000000000000000000000001
ml-storage-ser27.nmg01:
getfattr: Removing leading '/' from absolute path names
# file: data3/bricks/project2/371_37829/test-dir
trusted.gfid=0x57e0a8945a224ab4be2c6c71eada6217
trusted.glusterfs.dht=0xdbde0bb1000000001249249215555554
trusted.glusterfs.quota.dirty=0x3000
trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1=0x000000000000000000000000000000000000000000000001
trusted.glusterfs.quota.size.1=0x000000000000000000000000000000000000000000000001
ml-storage-ser20.nmg01:
getfattr: Removing leading '/' from absolute path names
# file: data3/bricks/project2/371_37829/test-dir
trusted.gfid=0x57e0a8945a224ab4be2c6c71eada6217
trusted.glusterfs.dht=0xdbde0bb100000000d8618615db6db6d7
trusted.glusterfs.quota.dirty=0x3000
trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1=0x000000000000000000000000000000000000000000000001
trusted.glusterfs.quota.size.1=0x000000000000000000000000000000000000000000000001
[root at ml-ctl-ser01 sunyuyusun]$ salt "ml-storage-ser2*.nmg01"
cmd.run
"getfattr -d -m. -e hex /data4/bricks/project2/371_37829/test-dir"
ml-storage-ser25.nmg01:
getfattr: Removing leading '/' from absolute path names
# file: data4/bricks/project2/371_37829/test-dir
trusted.gfid=0x57e0a8945a224ab4be2c6c71eada6217
trusted.glusterfs.dht=0xdbde0bb10000000024924924279e79e6
trusted.glusterfs.quota.dirty=0x3000
trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1=0x000000000000000000000000000000000000000000000001
trusted.glusterfs.quota.size.1=0x000000000000000000000000000000000000000000000001
ml-storage-ser27.nmg01:
getfattr: Removing leading '/' from absolute path names
# file: data4/bricks/project2/371_37829/test-dir
trusted.gfid=0x57e0a8945a224ab4be2c6c71eada6217
trusted.glusterfs.dht=0xdbde0bb100000000279e79e72aaaaaa9
trusted.glusterfs.quota.dirty=0x3000
trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1=0x000000000000000000000000000000000000000000000001
trusted.glusterfs.quota.size.1=0x000000000000000000000000000000000000000000000001
ml-storage-ser28.nmg01:
getfattr: Removing leading '/' from absolute path names
# file: data4/bricks/project2/371_37829/test-dir
trusted.gfid=0x57e0a8945a224ab4be2c6c71eada6217
trusted.glusterfs.dht=0xdbde0bb1000000002aaaaaaa2db6db6c
trusted.glusterfs.quota.dirty=0x3000
trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1=0x000000000000000000000000000000000000000000000001
trusted.glusterfs.quota.size.1=0x000000000000000000000000000000000000000000000001
ml-storage-ser23.nmg01:
getfattr: Removing leading '/' from absolute path names
# file: data4/bricks/project2/371_37829/test-dir
trusted.gfid=0x57e0a8945a224ab4be2c6c71eada6217
trusted.glusterfs.dht=0xdbde0bb1000000002186186124924923
trusted.glusterfs.quota.dirty=0x3000
trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1=0x000000000000000000000000000000000000000000000001
trusted.glusterfs.quota.size.1=0x000000000000000000000000000000000000000000000001
ml-storage-ser26.nmg01:
getfattr: Removing leading '/' from absolute path names
# file: data4/bricks/project2/371_37829/test-dir
trusted.gfid=0x57e0a8945a224ab4be2c6c71eada6217
trusted.glusterfs.dht=0xdbde0bb100000000279e79e72aaaaaa9
trusted.glusterfs.quota.dirty=0x3000
trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1=0x000000000000000000000000000000000000000000000001
trusted.glusterfs.quota.size.1=0x000000000000000000000000000000000000000000000001
ml-storage-ser29.nmg01:
getfattr: Removing leading '/' from absolute path names
# file: data4/bricks/project2/371_37829/test-dir
trusted.gfid=0x57e0a8945a224ab4be2c6c71eada6217
trusted.glusterfs.dht=0xdbde0bb1000000002aaaaaaa2db6db6c
trusted.glusterfs.quota.dirty=0x3000
trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1=0x000000000000000000000000000000000000000000000001
trusted.glusterfs.quota.size.1=0x000000000000000000000000000000000000000000000001
ml-storage-ser22.nmg01:
getfattr: Removing leading '/' from absolute path names
# file: data4/bricks/project2/371_37829/test-dir
trusted.gfid=0x57e0a8945a224ab4be2c6c71eada6217
trusted.glusterfs.dht=0xdbde0bb1000000002186186124924923
trusted.glusterfs.quota.dirty=0x3000
trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1=0x000000000000000000000000000000000000000000000001
trusted.glusterfs.quota.size.1=0x000000000000000000000000000000000000000000000001
ml-storage-ser24.nmg01:
getfattr: Removing leading '/' from absolute path names
# file: data4/bricks/project2/371_37829/test-dir
trusted.gfid=0x57e0a8945a224ab4be2c6c71eada6217
trusted.glusterfs.dht=0xdbde0bb10000000024924924279e79e6
trusted.glusterfs.quota.dirty=0x3000
trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1=0x000000000000000000000000000000000000000000000001
trusted.glusterfs.quota.size.1=0x000000000000000000000000000000000000000000000001
ml-storage-ser20.nmg01:
getfattr: Removing leading '/' from absolute path names
# file: data4/bricks/project2/371_37829/test-dir
c=0x57e0a8945a224ab4be2c6c71eada6217
trusted.glusterfs.dht=0xdbde0bb1000000001e79e79e21861860
trusted.glusterfs.quota.dirty=0x3000
trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1=0x000000000000000000000000000000000000000000000001
trusted.glusterfs.quota.size.1=0x000000000000000000000000000000000000000000000001
ml-storage-ser21.nmg01:
getfattr: Removing leading '/' from absolute path names
# file: data4/bricks/project2/371_37829/test-dir
trusted.gfid=0x57e0a8945a224ab4be2c6c71eada6217
trusted.glusterfs.dht=0xdbde0bb1000000001e79e79e21861860
trusted.glusterfs.quota.dirty=0x3000
trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1=0x000000000000000000000000000000000000000000000001
trusted.glusterfs.quota.size.1=0x000000000000000000000000000000000000000000000001
[root at ml-ctl-ser01 sunyuyusun]$
Is the infomation listed above what you need? I can see the two directory
(src and dst) are of diffenent trusted.gfid.
also I have another test and basically certain that when i enable quota,
this will happen in one mount point(i only mount the test volume in one
mount point)
Best Regards
Yours
Yu
Raghavendra Gowdappa <rgowdapp at redhat.com> ?2018?9?5??? ??12:40???
>
>
> On Wed, Sep 5, 2018 at 9:21 AM, yu sun <sunyu1949 at gmail.com>
wrote:
>
>> thank you , I will investigate and have a try.
>>
>
> Please get all the extended attributes of src and dst from all bricks.
> That should help us to (dis)prove whether its same bug.
>
>
>> BYW, I found another thing:
>> 1. I found this only occur when I try to mv dir to a subdir.
>> 2. I create a similar volume with default option and without quota
>> enabled, i found its ok, mv seems no problem.
>>
>> so I think it's possibly the volume option led to this problem, but
I
>> dont know which options and how to resolve this, the volume have about
25T
>> data.
>>
>> Best Regards.
>>
>> Raghavendra Gowdappa <rgowdapp at redhat.com> ?2018?9?5???
??10:50???
>>
>>>
>>>
>>> On Tue, Sep 4, 2018 at 5:28 PM, yu sun <sunyu1949 at
gmail.com> wrote:
>>>
>>>> Hi all:
>>>>
>>>> I have a replicated volume project2 with info:
>>>> Volume Name: project2 Type: Distributed-Replicate Volume ID:
>>>> 60175b8e-de0e-4409-81ae-7bb5eb5cacbf Status: Started Snapshot
Count: 0
>>>> Number of Bricks: 84 x 2 = 168 Transport-type: tcp Bricks:
Brick1:
>>>> node20:/data2/bricks/project2 Brick2:
node21:/data2/bricks/project2 Brick3:
>>>> node22:/data2/bricks/project2 Brick4:
node23:/data2/bricks/project2 Brick5:
>>>> node24:/data2/bricks/project2 Brick6:
node25:/data2/bricks/project2 Brick7:
>>>> node26:/data2/bricks/project2 Brick8:
node27:/data2/bricks/project2 Brick9:
>>>> node28:/data2/bricks/project2 Brick10:
node29:/data2/bricks/project2
>>>> Brick11: node30:/data2/bricks/project2 Brick12:
>>>> node31:/data2/bricks/project2 Brick13:
node32:/data2/bricks/project2
>>>> Brick14: node33:/data2/bricks/project2 Brick15:
>>>> node20:/data3/bricks/project2 Brick16:
node21:/data3/bricks/project2
>>>> Brick17: node22:/data3/bricks/project2 Brick18:
>>>> node23:/data3/bricks/project2 Brick19:
node24:/data3/bricks/project2
>>>> Brick20: node25:/data3/bricks/project2 Brick21:
>>>> node26:/data3/bricks/project2 Brick22:
node27:/data3/bricks/project2
>>>> Brick23: node28:/data3/bricks/project2 Brick24:
>>>> node29:/data3/bricks/project2 Brick25:
node30:/data3/bricks/project2
>>>> Brick26: node31:/data3/bricks/project2 Brick27:
>>>> node32:/data3/bricks/project2 Brick28:
node33:/data3/bricks/project2
>>>> Brick29: node20:/data4/bricks/project2 Brick30:
>>>> node21:/data4/bricks/project2 Brick31:
node22:/data4/bricks/project2
>>>> Brick32: node23:/data4/bricks/project2 Brick33:
>>>> node24:/data4/bricks/project2 Brick34:
node25:/data4/bricks/project2
>>>> Brick35: node26:/data4/bricks/project2 Brick36:
>>>> node27:/data4/bricks/project2 Brick37:
node28:/data4/bricks/project2
>>>> Brick38: node29:/data4/bricks/project2 Brick39:
>>>> node30:/data4/bricks/project2 Brick40:
node31:/data4/bricks/project2
>>>> Brick41: node32:/data4/bricks/project2 Brick42:
>>>> node33:/data4/bricks/project2 Brick43:
node20:/data5/bricks/project2
>>>> Brick44: node21:/data5/bricks/project2 Brick45:
>>>> node22:/data5/bricks/project2 Brick46:
node23:/data5/bricks/project2
>>>> Brick47: node24:/data5/bricks/project2 Brick48:
>>>> node25:/data5/bricks/project2 Brick49:
node26:/data5/bricks/project2
>>>> Brick50: node27:/data5/bricks/project2 Brick51:
>>>> node28:/data5/bricks/project2 Brick52:
node29:/data5/bricks/project2
>>>> Brick53: node30:/data5/bricks/project2 Brick54:
>>>> node31:/data5/bricks/project2 Brick55:
node32:/data5/bricks/project2
>>>> Brick56: node33:/data5/bricks/project2 Brick57:
>>>> node20:/data6/bricks/project2 Brick58:
node21:/data6/bricks/project2
>>>> Brick59: node22:/data6/bricks/project2 Brick60:
>>>> node23:/data6/bricks/project2 Brick61:
node24:/data6/bricks/project2
>>>> Brick62: node25:/data6/bricks/project2 Brick63:
>>>> node26:/data6/bricks/project2 Brick64:
node27:/data6/bricks/project2
>>>> Brick65: node28:/data6/bricks/project2 Brick66:
>>>> node29:/data6/bricks/project2 Brick67:
node30:/data6/bricks/project2
>>>> Brick68: node31:/data6/bricks/project2 Brick69:
>>>> node32:/data6/bricks/project2 Brick70:
node33:/data6/bricks/project2
>>>> Brick71: node20:/data7/bricks/project2 Brick72:
>>>> node21:/data7/bricks/project2 Brick73:
node22:/data7/bricks/project2
>>>> Brick74: node23:/data7/bricks/project2 Brick75:
>>>> node24:/data7/bricks/project2 Brick76:
node25:/data7/bricks/project2
>>>> Brick77: node26:/data7/bricks/project2 Brick78:
>>>> node27:/data7/bricks/project2 Brick79:
node28:/data7/bricks/project2
>>>> Brick80: node29:/data7/bricks/project2 Brick81:
>>>> node30:/data7/bricks/project2 Brick82:
node31:/data7/bricks/project2
>>>> Brick83: node32:/data7/bricks/project2 Brick84:
>>>> node33:/data7/bricks/project2 Brick85:
node20:/data8/bricks/project2
>>>> Brick86: node21:/data8/bricks/project2 Brick87:
>>>> node22:/data8/bricks/project2 Brick88:
node23:/data8/bricks/project2
>>>> Brick89: node24:/data8/bricks/project2 Brick90:
>>>> node25:/data8/bricks/project2 Brick91:
node26:/data8/bricks/project2
>>>> Brick92: node27:/data8/bricks/project2 Brick93:
>>>> node28:/data8/bricks/project2 Brick94:
node29:/data8/bricks/project2
>>>> Brick95: node30:/data8/bricks/project2 Brick96:
>>>> node31:/data8/bricks/project2 Brick97:
node32:/data8/bricks/project2
>>>> Brick98: node33:/data8/bricks/project2 Brick99:
>>>> node20:/data9/bricks/project2 Brick100:
node21:/data9/bricks/project2
>>>> Brick101: node22:/data9/bricks/project2 Brick102:
>>>> node23:/data9/bricks/project2 Brick103:
node24:/data9/bricks/project2
>>>> Brick104: node25:/data9/bricks/project2 Brick105:
>>>> node26:/data9/bricks/project2 Brick106:
node27:/data9/bricks/project2
>>>> Brick107: node28:/data9/bricks/project2 Brick108:
>>>> node29:/data9/bricks/project2 Brick109:
node30:/data9/bricks/project2
>>>> Brick110: node31:/data9/bricks/project2 Brick111:
>>>> node32:/data9/bricks/project2 Brick112:
node33:/data9/bricks/project2
>>>> Brick113: node20:/data10/bricks/project2 Brick114:
>>>> node21:/data10/bricks/project2 Brick115:
node22:/data10/bricks/project2
>>>> Brick116: node23:/data10/bricks/project2 Brick117:
>>>> node24:/data10/bricks/project2 Brick118:
node25:/data10/bricks/project2
>>>> Brick119: node26:/data10/bricks/project2 Brick120:
>>>> node27:/data10/bricks/project2 Brick121:
node28:/data10/bricks/project2
>>>> Brick122: node29:/data10/bricks/project2 Brick123:
>>>> node30:/data10/bricks/project2 Brick124:
node31:/data10/bricks/project2
>>>> Brick125: node32:/data10/bricks/project2 Brick126:
>>>> node33:/data10/bricks/project2 Brick127:
node20:/data11/bricks/project2
>>>> Brick128: node21:/data11/bricks/project2 Brick129:
>>>> node22:/data11/bricks/project2 Brick130:
node23:/data11/bricks/project2
>>>> Brick131: node24:/data11/bricks/project2 Brick132:
>>>> node25:/data11/bricks/project2 Brick133:
node26:/data11/bricks/project2
>>>> Brick134: node27:/data11/bricks/project2 Brick135:
>>>> node28:/data11/bricks/project2 Brick136:
node29:/data11/bricks/project2
>>>> Brick137: node30:/data11/bricks/project2 Brick138:
>>>> node31:/data11/bricks/project2 Brick139:
node32:/data11/bricks/project2
>>>> Brick140: node33:/data11/bricks/project2 Brick141:
>>>> node20:/data12/bricks/project2 Brick142:
node21:/data12/bricks/project2
>>>> Brick143: node22:/data12/bricks/project2 Brick144:
>>>> node23:/data12/bricks/project2 Brick145:
node24:/data12/bricks/project2
>>>> Brick146: node25:/data12/bricks/project2 Brick147:
>>>> node26:/data12/bricks/project2 Brick148:
node27:/data12/bricks/project2
>>>> Brick149: node28:/data12/bricks/project2 Brick150:
>>>> node29:/data12/bricks/project2 Brick151:
node30:/data12/bricks/project2
>>>> Brick152: node31:/data12/bricks/project2 Brick153:
>>>> node32:/data12/bricks/project2 Brick154:
node33:/data12/bricks/project2
>>>> Brick155: node20:/data13/bricks/project2 Brick156:
>>>> node21:/data13/bricks/project2 Brick157:
node22:/data13/bricks/project2
>>>> Brick158: node23:/data13/bricks/project2 Brick159:
>>>> node24:/data13/bricks/project2 Brick160:
node25:/data13/bricks/project2
>>>> Brick161: node26:/data13/bricks/project2 Brick162:
>>>> node27:/data13/bricks/project2 Brick163:
node28:/data13/bricks/project2
>>>> Brick164: node29:/data13/bricks/project2 Brick165:
>>>> node30:/data13/bricks/project2 Brick166:
node31:/data13/bricks/project2
>>>> Brick167: node32:/data13/bricks/project2 Brick168:
>>>> node33:/data13/bricks/project2 Options Reconfigured:
>>>> performance.force-readdirp: on performance.write-behind: off
>>>> performance.stat-prefetch: on performance.client-io-threads: on
>>>> nfs.disable: on transport.address-family: inet features.quota:
on
>>>> features.inode-quota: on features.quota-deem-statfs: on
>>>> cluster.readdir-optimize: on cluster.lookup-optimize: on
>>>> dht.force-readdirp: off client.event-threads: 10
server.event-threads: 10
>>>> performance.readdir-ahead: on performance.io-cache: on
>>>> performance.flush-behind: on performance.cache-size: 5GB
>>>> performance.cache-max-file-size: 1MB
performance.write-behind-window-size:
>>>> 10MB performance.read-ahead: off network.remote-dio: enable
>>>> performance.strict-o-direct: disable
performance.io-thread-count: 25
>>>>
>>>>
>>>> the volume looks ok, and I mount this volume on my client
machine:
>>>> mount -t glusterfs -o oom-score-adj=-999 -o
direct-io-mode=disable -o
>>>> use-readdirp=no node20:/project2 /mnt/project2
>>>>
>>>> I have a directory in /mnt/project2/, but when I mv the
directory to
>>>> other dirs, files in the dir lost while tree or ls the dir,
some files
>>>> missing, my operations is list as below:
>>>>
>>>
>>> Looks very similar to:
>>> https://bugzilla.redhat.com/show_bug.cgi?id=1118762
>>> https://bugzilla.redhat.com/show_bug.cgi?id=1337394
>>>
>>>
>>>
>>>> root at ml-gpu-ser129.nmg01:/mnt/project2/371_37829$ mkdir
test-dir
>>>> root at ml-gpu-ser129.nmg01:/mnt/project2/371_37829$ tree
>>>> .
>>>> ??? face_landmarks
>>>> ? ??? alive
>>>> ? ??? logs_20180823_28
>>>> ? ??? info_000000.out
>>>> ? ??? info_000001.out
>>>> ? ??? info_000002.out
>>>> ? ??? info_000003.out
>>>> ? ??? info_000004.out
>>>> ? ??? info_000005.out
>>>> ? ??? info_000006.out
>>>> ? ??? info_000007.out
>>>> ? ??? info_000008.out
>>>> ? ??? info_000009.out
>>>> ? ??? info_000010.out
>>>> ? ??? info_000011.out
>>>> ? ??? info_000012.out
>>>> ? ??? info_000013.out
>>>> ? ??? info_000014.out
>>>> ? ??? info_000015.out
>>>> ? ??? info_000016.out
>>>> ? ??? info_000017.out
>>>> ? ??? info_000018.out
>>>> ? ??? info_000019.out
>>>> ??? test-dir
>>>>
>>>> 4 directories, 20 files
>>>> root at ml-gpu-ser129.nmg01:/mnt/project2/371_37829$ mv
face_landmarks/
>>>> test-dir/
>>>> root at ml-gpu-ser129.nmg01:/mnt/project2/371_37829$ tree
>>>> .
>>>> ??? test-dir
>>>> ??? face_landmarks
>>>>
>>>> 2 directories, 0 files
>>>> root at ml-gpu-ser129.nmg01:/mnt/project2/371_37829$ cd
>>>> test-dir/face_landmarks/
>>>> root at
ml-gpu-ser129.nmg01:/mnt/project2/371_37829/test-dir/face_landmarks$
>>>> ls
>>>> root at
ml-gpu-ser129.nmg01:/mnt/project2/371_37829/test-dir/face_landmarks$
>>>> cd ..
>>>> root at ml-gpu-ser129.nmg01:/mnt/project2/371_37829/test-dir$
mv
>>>> face_landmarks/ ..
>>>> root at ml-gpu-ser129.nmg01:/mnt/project2/371_37829/test-dir$
cd ..
>>>> root at ml-gpu-ser129.nmg01:/mnt/project2/371_37829$ tree
>>>> .
>>>> ??? face_landmarks
>>>> ? ??? alive
>>>> ??? test-dir
>>>>
>>>> 3 directories, 0 files
>>>> root at ml-gpu-ser129.nmg01:/mnt/project2/371_37829$
>>>>
>>>> I think i make some mistakes with volume option, buti i am not
sure, so
>>>> how can i find the lost files? the files seems still int the
directory,
>>>> because i cant remove the directory and rm tell me "Not
empty directory"
>>>>
>>>
>>> Its likely that src and dst of mv having same gfid and that's
causing
>>> the issues. Can you look into both src and dst paths on all bricks?
Union
>>> of contents of both directories should give all the files in the
src
>>> directory before mv. Once found you can,
>>> * keep a backup of contents of src and dst on all bricks
>>> * remove trusted.gfid xattr on src and dst from all bricks
>>> * remove gfid handle (.glusterfs/<first two characters of
gfid>/<second
>>> set of two characters of gfid>/<gfid> on each brick)
>>> * disable readdirplus in entire stack (maybe you can use a tmp
mount for
>>> this) [1]
>>> * stat src and dst on a mount point with readdirplus disabled.
>>> * Now you'll see two directories src and dst on mountpoint. You
can
>>> copy the contents of both into a new directory
>>>
>>> [1]
>>>
https://lists.gluster.org/pipermail/gluster-users/2017-March/030148.html
>>>
>>>
>>>>
>>>> Any suggestions is appreciated.
>>>> Many Thanks
>>>>
>>>> Best regards
>>>> Yu
>>>>
>>>> _______________________________________________
>>>> Gluster-users mailing list
>>>> Gluster-users at gluster.org
>>>> https://lists.gluster.org/mailman/listinfo/gluster-users
>>>>
>>>
>>>
>
-------------- next part --------------
An HTML attachment was scrubbed...
URL:
<http://lists.gluster.org/pipermail/gluster-users/attachments/20180905/c6d1da46/attachment.html>
Hi, Please try turning off cluster.readdir-optimize and see it if helps. If not, please send us the client mount logs and a tcpdump of when the *ls* is performed from the client. Use the following to capture the dump: tcpdump -i any -s 0 -w /var/tmp/dirls.pcap tcp and not port 22 Thanks, Nithya On 5 September 2018 at 12:46, yu sun <sunyu1949 at gmail.com> wrote:> > [root at ml-ctl-ser01 sunyuyusun]$ salt "ml-storage-ser2*.nmg01" cmd.run > "getfattr -d -m. -e hex /data2/bricks/project2/371_37829/face_landmarks" > ml-storage-ser24.nmg01: > getfattr: Removing leading '/' from absolute path names > # file: data2/bricks/project2/371_37829/face_landmarks > trusted.gfid=0x3764e261741d423d8c8eb6ad5b716429 > trusted.glusterfs.dht=0xdbde0bb100000000b0c30c2eb3cf3cf0 > trusted.glusterfs.quota.dirty=0x3000 > trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1> 0x00000000000d760000000000000000010000000000000003 > trusted.glusterfs.quota.size.1=0x00000000000d7600000000000000 > 00010000000000000003 > ml-storage-ser26.nmg01: > getfattr: Removing leading '/' from absolute path names > # file: data2/bricks/project2/371_37829/face_landmarks > trusted.gfid=0x3764e261741d423d8c8eb6ad5b716429 > trusted.glusterfs.dht=0xdbde0bb100000000d249248fd5555551 > trusted.glusterfs.quota.dirty=0x3000 > trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1> 0x000000000000000000000000000000000000000000000003 > trusted.glusterfs.quota.size.1=0x0000000000000000000000000000 > 00000000000000000003 > ml-storage-ser23.nmg01: > getfattr: Removing leading '/' from absolute path names > # file: data2/bricks/project2/371_37829/face_landmarks > trusted.gfid=0x3764e261741d423d8c8eb6ad5b716429 > trusted.glusterfs.dht=0xdbde0bb1000000008f3cf3cd9249248f > trusted.glusterfs.quota.dirty=0x3000 > trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1> 0x000000000000000000000000000000000000000000000003 > trusted.glusterfs.quota.size.1=0x0000000000000000000000000000 > 00000000000000000003 > ml-storage-ser27.nmg01: > getfattr: Removing leading '/' from absolute path names > # file: data2/bricks/project2/371_37829/face_landmarks > trusted.gfid=0x3764e261741d423d8c8eb6ad5b716429 > trusted.glusterfs.dht=0xdbde0bb100000000d249248fd5555551 > trusted.glusterfs.quota.dirty=0x3000 > trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1> 0x000000000000000000000000000000000000000000000003 > trusted.glusterfs.quota.size.1=0x0000000000000000000000000000 > 00000000000000000003 > ml-storage-ser25.nmg01: > getfattr: Removing leading '/' from absolute path names > # file: data2/bricks/project2/371_37829/face_landmarks > trusted.gfid=0x3764e261741d423d8c8eb6ad5b716429 > trusted.glusterfs.dht=0xdbde0bb100000000b0c30c2eb3cf3cf0 > trusted.glusterfs.quota.dirty=0x3000 > trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1> 0x00000000000d760000000000000000010000000000000003 > trusted.glusterfs.quota.size.1=0x00000000000d7600000000000000 > 00010000000000000003 > ml-storage-ser28.nmg01: > getfattr: Removing leading '/' from absolute path names > # file: data2/bricks/project2/371_37829/face_landmarks > trusted.gfid=0x3764e261741d423d8c8eb6ad5b716429 > trusted.glusterfs.dht=0xdbde0bb100000000f3cf3cf0f6db6db2 > trusted.glusterfs.quota.dirty=0x3000 > trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1> 0x000000000000000000000000000000000000000000000003 > trusted.glusterfs.quota.size.1=0x0000000000000000000000000000 > 00000000000000000003 > ml-storage-ser29.nmg01: > getfattr: Removing leading '/' from absolute path names > # file: data2/bricks/project2/371_37829/face_landmarks > trusted.gfid=0x3764e261741d423d8c8eb6ad5b716429 > trusted.glusterfs.dht=0xdbde0bb100000000f3cf3cf0f6db6db2 > trusted.glusterfs.quota.dirty=0x3000 > trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1> 0x000000000000000000000000000000000000000000000003 > trusted.glusterfs.quota.size.1=0x0000000000000000000000000000 > 00000000000000000003 > ml-storage-ser20.nmg01: > getfattr: Removing leading '/' from absolute path names > # file: data2/bricks/project2/371_37829/face_landmarks > trusted.gfid=0x3764e261741d423d8c8eb6ad5b716429 > trusted.glusterfs.dht=0xdbde0bb1000000008c30c30a8f3cf3cc > trusted.glusterfs.quota.dirty=0x3000 > trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1> 0x000000000000000000000000000000000000000000000003 > trusted.glusterfs.quota.size.1=0x0000000000000000000000000000 > 00000000000000000003 > ml-storage-ser22.nmg01: > getfattr: Removing leading '/' from absolute path names > # file: data2/bricks/project2/371_37829/face_landmarks > trusted.gfid=0x3764e261741d423d8c8eb6ad5b716429 > trusted.glusterfs.dht=0xdbde0bb1000000008f3cf3cd9249248f > trusted.glusterfs.quota.dirty=0x3000 > trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1> 0x000000000000000000000000000000000000000000000003 > trusted.glusterfs.quota.size.1=0x0000000000000000000000000000 > 00000000000000000003 > ml-storage-ser21.nmg01: > getfattr: Removing leading '/' from absolute path names > # file: data2/bricks/project2/371_37829/face_landmarks > trusted.gfid=0x3764e261741d423d8c8eb6ad5b716429 > trusted.glusterfs.dht=0xdbde0bb1000000008c30c30a8f3cf3cc > trusted.glusterfs.quota.dirty=0x3000 > trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1> 0x000000000000000000000000000000000000000000000003 > trusted.glusterfs.quota.size.1=0x0000000000000000000000000000 > 00000000000000000003 > [root at ml-ctl-ser01 sunyuyusun]$ salt "ml-storage-ser2*.nmg01" cmd.run > "getfattr -d -m. -e hex /data3/bricks/project2/371_37829/face_landmarks" > ml-storage-ser28.nmg01: > getfattr: Removing leading '/' from absolute path names > # file: data3/bricks/project2/371_37829/face_landmarks > trusted.gfid=0x3764e261741d423d8c8eb6ad5b716429 > trusted.glusterfs.dht=0xdbde0bb1000000009555555398618615 > trusted.glusterfs.quota.dirty=0x3000 > trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1> 0x00000000001aec0000000000000000020000000000000003 > trusted.glusterfs.quota.size.1=0x00000000001aec00000000000000 > 00020000000000000003 > ml-storage-ser25.nmg01: > getfattr: Removing leading '/' from absolute path names > # file: data3/bricks/project2/371_37829/face_landmarks > trusted.gfid=0x3764e261741d423d8c8eb6ad5b716429 > trusted.glusterfs.dht=0xdbde0bb100000000892492478c30c309 > trusted.glusterfs.quota.dirty=0x3000 > trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1> 0x000000000000000000000000000000000000000000000003 > trusted.glusterfs.quota.size.1=0x0000000000000000000000000000 > 00000000000000000003 > ml-storage-ser24.nmg01: > getfattr: Removing leading '/' from absolute path names > # file: data3/bricks/project2/371_37829/face_landmarks > trusted.gfid=0x3764e261741d423d8c8eb6ad5b716429 > trusted.glusterfs.dht=0xdbde0bb100000000892492478c30c309 > trusted.glusterfs.quota.dirty=0x3000 > trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1> 0x000000000000000000000000000000000000000000000003 > trusted.glusterfs.quota.size.1=0x0000000000000000000000000000 > 00000000000000000003 > ml-storage-ser27.nmg01: > getfattr: Removing leading '/' from absolute path names > # file: data3/bricks/project2/371_37829/face_landmarks > trusted.gfid=0x3764e261741d423d8c8eb6ad5b716429 > trusted.glusterfs.dht=0xdbde0bb1000000009249249095555552 > trusted.glusterfs.quota.dirty=0x3000 > trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1> 0x000000000000000000000000000000000000000000000003 > trusted.glusterfs.quota.size.1=0x0000000000000000000000000000 > 00000000000000000003 > ml-storage-ser23.nmg01: > getfattr: Removing leading '/' from absolute path names > # file: data3/bricks/project2/371_37829/face_landmarks > trusted.gfid=0x3764e261741d423d8c8eb6ad5b716429 > trusted.glusterfs.dht=0xdbde0bb10000000079e79e787cf3cf3a > trusted.glusterfs.quota.dirty=0x3000 > trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1> 0x000000000000000000000000000000000000000000000003 > trusted.glusterfs.quota.size.1=0x0000000000000000000000000000 > 00000000000000000003 > ml-storage-ser22.nmg01: > getfattr: Removing leading '/' from absolute path names > # file: data3/bricks/project2/371_37829/face_landmarks > trusted.gfid=0x3764e261741d423d8c8eb6ad5b716429 > trusted.glusterfs.dht=0xdbde0bb10000000079e79e787cf3cf3a > trusted.glusterfs.quota.dirty=0x3000 > trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1> 0x000000000000000000000000000000000000000000000003 > trusted.glusterfs.quota.size.1=0x0000000000000000000000000000 > 00000000000000000003 > ml-storage-ser29.nmg01: > getfattr: Removing leading '/' from absolute path names > # file: data3/bricks/project2/371_37829/face_landmarks > trusted.gfid=0x3764e261741d423d8c8eb6ad5b716429 > trusted.glusterfs.dht=0xdbde0bb1000000009555555398618615 > trusted.glusterfs.quota.dirty=0x3000 > trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1> 0x00000000001aec0000000000000000020000000000000003 > trusted.glusterfs.quota.size.1=0x00000000001aec00000000000000 > 00020000000000000003 > ml-storage-ser26.nmg01: > getfattr: Removing leading '/' from absolute path names > # file: data3/bricks/project2/371_37829/face_landmarks > trusted.gfid=0x3764e261741d423d8c8eb6ad5b716429 > trusted.glusterfs.dht=0xdbde0bb1000000009249249095555552 > trusted.glusterfs.quota.dirty=0x3000 > trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1> 0x000000000000000000000000000000000000000000000003 > trusted.glusterfs.quota.size.1=0x0000000000000000000000000000 > 00000000000000000003 > ml-storage-ser21.nmg01: > getfattr: Removing leading '/' from absolute path names > # file: data3/bricks/project2/371_37829/face_landmarks > trusted.gfid=0x3764e261741d423d8c8eb6ad5b716429 > trusted.glusterfs.dht=0xdbde0bb100000000586186175b6db6d9 > trusted.glusterfs.quota.dirty=0x3000 > trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1> 0x000000000000000000000000000000000000000000000003 > trusted.glusterfs.quota.size.1=0x0000000000000000000000000000 > 00000000000000000003 > ml-storage-ser20.nmg01: > getfattr: Removing leading '/' from absolute path names > # file: data3/bricks/project2/371_37829/face_landmarks > trusted.gfid=0x3764e261741d423d8c8eb6ad5b716429 > trusted.glusterfs.dht=0xdbde0bb100000000586186175b6db6d9 > trusted.glusterfs.quota.dirty=0x3000 > trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1> 0x000000000000000000000000000000000000000000000003 > trusted.glusterfs.quota.size.1=0x0000000000000000000000000000 > 00000000000000000003 > > > > [root at ml-ctl-ser01 sunyuyusun]$ salt "ml-storage-ser2*.nmg01" cmd.run > "getfattr -d -m. -e hex /data4/bricks/project2/371_37829/face_landmarks" > ml-storage-ser25.nmg01: > getfattr: Removing leading '/' from absolute path names > # file: data4/bricks/project2/371_37829/face_landmarks > trusted.gfid=0x3764e261741d423d8c8eb6ad5b716429 > trusted.glusterfs.dht=0xdbde0bb100000000a4924922a79e79e4 > trusted.glusterfs.quota.dirty=0x3000 > trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1> 0x000000000000000000000000000000000000000000000003 > trusted.glusterfs.quota.size.1=0x0000000000000000000000000000 > 00000000000000000003 > ml-storage-ser23.nmg01: > getfattr: Removing leading '/' from absolute path names > # file: data4/bricks/project2/371_37829/face_landmarks > trusted.gfid=0x3764e261741d423d8c8eb6ad5b716429 > trusted.glusterfs.dht=0xdbde0bb100000000a186185fa4924921 > trusted.glusterfs.quota.dirty=0x3000 > trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1> 0x000000000000000000000000000000000000000000000003 > trusted.glusterfs.quota.size.1=0x0000000000000000000000000000 > 00000000000000000003 > ml-storage-ser24.nmg01: > getfattr: Removing leading '/' from absolute path names > # file: data4/bricks/project2/371_37829/face_landmarks > trusted.gfid=0x3764e261741d423d8c8eb6ad5b716429 > trusted.glusterfs.dht=0xdbde0bb100000000a4924922a79e79e4 > trusted.glusterfs.quota.dirty=0x3000 > trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1> 0x000000000000000000000000000000000000000000000003 > trusted.glusterfs.quota.size.1=0x0000000000000000000000000000 > 00000000000000000003 > ml-storage-ser28.nmg01: > getfattr: Removing leading '/' from absolute path names > # file: data4/bricks/project2/371_37829/face_landmarks > trusted.gfid=0x3764e261741d423d8c8eb6ad5b716429 > trusted.glusterfs.dht=0xdbde0bb100000000aaaaaaa8adb6db6a > trusted.glusterfs.quota.dirty=0x3000 > trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1> 0x000000000000000000000000000000000000000000000003 > trusted.glusterfs.quota.size.1=0x0000000000000000000000000000 > 00000000000000000003 > ml-storage-ser27.nmg01: > getfattr: Removing leading '/' from absolute path names > # file: data4/bricks/project2/371_37829/face_landmarks > trusted.gfid=0x3764e261741d423d8c8eb6ad5b716429 > trusted.glusterfs.dht=0xdbde0bb100000000a79e79e5aaaaaaa7 > trusted.glusterfs.quota.dirty=0x3000 > trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1> 0x000000000000000000000000000000000000000000000003 > trusted.glusterfs.quota.size.1=0x0000000000000000000000000000 > 00000000000000000003 > ml-storage-ser26.nmg01: > getfattr: Removing leading '/' from absolute path names > # file: data4/bricks/project2/371_37829/face_landmarks > trusted.gfid=0x3764e261741d423d8c8eb6ad5b716429 > trusted.glusterfs.dht=0xdbde0bb100000000a79e79e5aaaaaaa7 > trusted.glusterfs.quota.dirty=0x3000 > trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1> 0x000000000000000000000000000000000000000000000003 > trusted.glusterfs.quota.size.1=0x0000000000000000000000000000 > 00000000000000000003 > ml-storage-ser21.nmg01: > getfattr: Removing leading '/' from absolute path names > # file: data4/bricks/project2/371_37829/face_landmarks > trusted.gfid=0x3764e261741d423d8c8eb6ad5b716429 > trusted.glusterfs.dht=0xdbde0bb1000000009e79e79ca186185e > trusted.glusterfs.quota.dirty=0x3000 > trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1> 0x000000000000000000000000000000000000000000000003 > trusted.glusterfs.quota.size.1=0x0000000000000000000000000000 > 00000000000000000003 > ml-storage-ser22.nmg01: > getfattr: Removing leading '/' from absolute path names > # file: data4/bricks/project2/371_37829/face_landmarks > trusted.gfid=0x3764e261741d423d8c8eb6ad5b716429 > trusted.glusterfs.dht=0xdbde0bb100000000a186185fa4924921 > trusted.glusterfs.quota.dirty=0x3000 > trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1> 0x000000000000000000000000000000000000000000000003 > trusted.glusterfs.quota.size.1=0x0000000000000000000000000000 > 00000000000000000003 > ml-storage-ser29.nmg01: > getfattr: Removing leading '/' from absolute path names > # file: data4/bricks/project2/371_37829/face_landmarks > trusted.gfid=0x3764e261741d423d8c8eb6ad5b716429 > trusted.glusterfs.dht=0xdbde0bb100000000aaaaaaa8adb6db6a > trusted.glusterfs.quota.dirty=0x3000 > trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1> 0x000000000000000000000000000000000000000000000003 > trusted.glusterfs.quota.size.1=0x0000000000000000000000000000 > 00000000000000000003 > ml-storage-ser20.nmg01: > getfattr: Removing leading '/' from absolute path names > # file: data4/bricks/project2/371_37829/face_landmarks > trusted.gfid=0x3764e261741d423d8c8eb6ad5b716429 > trusted.glusterfs.dht=0xdbde0bb1000000009e79e79ca186185e > trusted.glusterfs.quota.dirty=0x3000 > trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1> 0x000000000000000000000000000000000000000000000003 > trusted.glusterfs.quota.size.1=0x0000000000000000000000000000 > 00000000000000000003 > > > > > [root at ml-ctl-ser01 sunyuyusun]$ salt "ml-storage-ser2*.nmg01" cmd.run > "getfattr -d -m. -e hex /data2/bricks/project2/371_37829/test-dir" > ml-storage-ser23.nmg01: > getfattr: Removing leading '/' from absolute path names > # file: data2/bricks/project2/371_37829/test-dir > trusted.gfid=0x57e0a8945a224ab4be2c6c71eada6217 > trusted.glusterfs.dht=0xdbde0bb1000000000f3cf3cf12492491 > trusted.glusterfs.quota.dirty=0x3000 > trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1> 0x000000000000000000000000000000000000000000000001 > trusted.glusterfs.quota.size.1=0x0000000000000000000000000000 > 00000000000000000001 > ml-storage-ser24.nmg01: > getfattr: Removing leading '/' from absolute path names > # file: data2/bricks/project2/371_37829/test-dir > trusted.gfid=0x57e0a8945a224ab4be2c6c71eada6217 > trusted.glusterfs.dht=0xdbde0bb10000000030c30c3033cf3cf2 > trusted.glusterfs.quota.dirty=0x3000 > trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1> 0x000000000000000000000000000000000000000000000001 > trusted.glusterfs.quota.size.1=0x0000000000000000000000000000 > 00000000000000000001 > ml-storage-ser28.nmg01: > getfattr: Removing leading '/' from absolute path names > # file: data2/bricks/project2/371_37829/test-dir > trusted.gfid=0x57e0a8945a224ab4be2c6c71eada6217 > trusted.glusterfs.dht=0xdbde0bb10000000073cf3cf276db6db4 > trusted.glusterfs.quota.dirty=0x3000 > trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1> 0x000000000000000000000000000000000000000000000001 > trusted.glusterfs.quota.size.1=0x0000000000000000000000000000 > 00000000000000000001 > ml-storage-ser25.nmg01: > getfattr: Removing leading '/' from absolute path names > # file: data2/bricks/project2/371_37829/test-dir > trusted.gfid=0x57e0a8945a224ab4be2c6c71eada6217 > trusted.glusterfs.dht=0xdbde0bb10000000030c30c3033cf3cf2 > trusted.glusterfs.quota.dirty=0x3000 > trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1> 0x000000000000000000000000000000000000000000000001 > trusted.glusterfs.quota.size.1=0x0000000000000000000000000000 > 00000000000000000001 > ml-storage-ser26.nmg01: > getfattr: Removing leading '/' from absolute path names > # file: data2/bricks/project2/371_37829/test-dir > trusted.gfid=0x57e0a8945a224ab4be2c6c71eada6217 > trusted.glusterfs.dht=0xdbde0bb1000000005249249155555553 > trusted.glusterfs.quota.dirty=0x3000 > trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1> 0x000000000000000000000000000000000000000000000001 > trusted.glusterfs.quota.size.1=0x0000000000000000000000000000 > 00000000000000000001 > ml-storage-ser27.nmg01: > getfattr: Removing leading '/' from absolute path names > # file: data2/bricks/project2/371_37829/test-dir > trusted.gfid=0x57e0a8945a224ab4be2c6c71eada6217 > trusted.glusterfs.dht=0xdbde0bb1000000005249249155555553 > trusted.glusterfs.quota.dirty=0x3000 > trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1> 0x000000000000000000000000000000000000000000000001 > trusted.glusterfs.quota.size.1=0x0000000000000000000000000000 > 00000000000000000001 > ml-storage-ser22.nmg01: > getfattr: Removing leading '/' from absolute path names > # file: data2/bricks/project2/371_37829/test-dir > trusted.gfid=0x57e0a8945a224ab4be2c6c71eada6217 > trusted.glusterfs.dht=0xdbde0bb1000000000f3cf3cf12492491 > trusted.glusterfs.quota.dirty=0x3000 > trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1> 0x000000000000000000000000000000000000000000000001 > trusted.glusterfs.quota.size.1=0x0000000000000000000000000000 > 00000000000000000001 > ml-storage-ser29.nmg01: > getfattr: Removing leading '/' from absolute path names > # file: data2/bricks/project2/371_37829/test-dir > trusted.gfid=0x57e0a8945a224ab4be2c6c71eada6217 > trusted.glusterfs.dht=0xdbde0bb10000000073cf3cf276db6db4 > trusted.glusterfs.quota.dirty=0x3000 > trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1> 0x000000000000000000000000000000000000000000000001 > trusted.glusterfs.quota.size.1=0x0000000000000000000000000000 > 00000000000000000001 > ml-storage-ser21.nmg01: > getfattr: Removing leading '/' from absolute path names > # file: data2/bricks/project2/371_37829/test-dir > trusted.gfid=0x57e0a8945a224ab4be2c6c71eada6217 > trusted.glusterfs.dht=0xdbde0bb1000000000c30c30c0f3cf3ce > trusted.glusterfs.quota.dirty=0x3000 > trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1> 0x000000000000000000000000000000000000000000000001 > trusted.glusterfs.quota.size.1=0x0000000000000000000000000000 > 00000000000000000001 > ml-storage-ser20.nmg01: > getfattr: Removing leading '/' from absolute path names > # file: data2/bricks/project2/371_37829/test-dir > trusted.gfid=0x57e0a8945a224ab4be2c6c71eada6217 > trusted.glusterfs.dht=0xdbde0bb1000000000c30c30c0f3cf3ce > trusted.glusterfs.quota.dirty=0x3000 > trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1> 0x000000000000000000000000000000000000000000000001 > trusted.glusterfs.quota.size.1=0x0000000000000000000000000000 > 00000000000000000001 > > > > [root at ml-ctl-ser01 sunyuyusun]$ salt "ml-storage-ser2*.nmg01" cmd.run > "getfattr -d -m. -e hex /data3/bricks/project2/371_37829/test-dir" > ml-storage-ser23.nmg01: > getfattr: Removing leading '/' from absolute path names > # file: data3/bricks/project2/371_37829/test-dir > trusted.gfid=0x57e0a8945a224ab4be2c6c71eada6217 > trusted.glusterfs.dht=0xdbde0bb100000000f9e79e76fcf3cf38 > trusted.glusterfs.quota.dirty=0x3000 > trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1> 0x000000000000000000000000000000000000000000000001 > trusted.glusterfs.quota.size.1=0x0000000000000000000000000000 > 00000000000000000001 > ml-storage-ser26.nmg01: > getfattr: Removing leading '/' from absolute path names > # file: data3/bricks/project2/371_37829/test-dir > trusted.gfid=0x57e0a8945a224ab4be2c6c71eada6217 > trusted.glusterfs.dht=0xdbde0bb1000000001249249215555554 > trusted.glusterfs.quota.dirty=0x3000 > trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1> 0x000000000000000000000000000000000000000000000001 > trusted.glusterfs.quota.size.1=0x0000000000000000000000000000 > 00000000000000000001 > ml-storage-ser25.nmg01: > getfattr: Removing leading '/' from absolute path names > # file: data3/bricks/project2/371_37829/test-dir > trusted.gfid=0x57e0a8945a224ab4be2c6c71eada6217 > trusted.glusterfs.dht=0xdbde0bb100000000092492490c30c30b > trusted.glusterfs.quota.dirty=0x3000 > trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1> 0x000000000000000000000000000000000000000000000001 > trusted.glusterfs.quota.size.1=0x0000000000000000000000000000 > 00000000000000000001 > ml-storage-ser28.nmg01: > getfattr: Removing leading '/' from absolute path names > # file: data3/bricks/project2/371_37829/test-dir > trusted.gfid=0x57e0a8945a224ab4be2c6c71eada6217 > trusted.glusterfs.dht=0xdbde0bb1000000001555555518618617 > trusted.glusterfs.quota.dirty=0x3000 > trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1> 0x000000000000000000000000000000000000000000000001 > trusted.glusterfs.quota.size.1=0x0000000000000000000000000000 > 00000000000000000001 > ml-storage-ser24.nmg01: > getfattr: Removing leading '/' from absolute path names > # file: data3/bricks/project2/371_37829/test-dir > trusted.gfid=0x57e0a8945a224ab4be2c6c71eada6217 > trusted.glusterfs.dht=0xdbde0bb100000000092492490c30c30b > trusted.glusterfs.quota.dirty=0x3000 > trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1> 0x000000000000000000000000000000000000000000000001 > trusted.glusterfs.quota.size.1=0x0000000000000000000000000000 > 00000000000000000001 > ml-storage-ser22.nmg01: > getfattr: Removing leading '/' from absolute path names > # file: data3/bricks/project2/371_37829/test-dir > trusted.gfid=0x57e0a8945a224ab4be2c6c71eada6217 > trusted.glusterfs.dht=0xdbde0bb100000000f9e79e76fcf3cf38 > trusted.glusterfs.quota.dirty=0x3000 > trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1> 0x000000000000000000000000000000000000000000000001 > trusted.glusterfs.quota.size.1=0x0000000000000000000000000000 > 00000000000000000001 > ml-storage-ser29.nmg01: > getfattr: Removing leading '/' from absolute path names > # file: data3/bricks/project2/371_37829/test-dir > trusted.gfid=0x57e0a8945a224ab4be2c6c71eada6217 > trusted.glusterfs.dht=0xdbde0bb1000000001555555518618617 > trusted.glusterfs.quota.dirty=0x3000 > trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1> 0x000000000000000000000000000000000000000000000001 > trusted.glusterfs.quota.size.1=0x0000000000000000000000000000 > 00000000000000000001 > ml-storage-ser21.nmg01: > getfattr: Removing leading '/' from absolute path names > # file: data3/bricks/project2/371_37829/test-dir > trusted.gfid=0x57e0a8945a224ab4be2c6c71eada6217 > trusted.glusterfs.dht=0xdbde0bb100000000d8618615db6db6d7 > trusted.glusterfs.quota.dirty=0x3000 > trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1> 0x000000000000000000000000000000000000000000000001 > trusted.glusterfs.quota.size.1=0x0000000000000000000000000000 > 00000000000000000001 > ml-storage-ser27.nmg01: > getfattr: Removing leading '/' from absolute path names > # file: data3/bricks/project2/371_37829/test-dir > trusted.gfid=0x57e0a8945a224ab4be2c6c71eada6217 > trusted.glusterfs.dht=0xdbde0bb1000000001249249215555554 > trusted.glusterfs.quota.dirty=0x3000 > trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1> 0x000000000000000000000000000000000000000000000001 > trusted.glusterfs.quota.size.1=0x0000000000000000000000000000 > 00000000000000000001 > ml-storage-ser20.nmg01: > getfattr: Removing leading '/' from absolute path names > # file: data3/bricks/project2/371_37829/test-dir > trusted.gfid=0x57e0a8945a224ab4be2c6c71eada6217 > trusted.glusterfs.dht=0xdbde0bb100000000d8618615db6db6d7 > trusted.glusterfs.quota.dirty=0x3000 > trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1> 0x000000000000000000000000000000000000000000000001 > trusted.glusterfs.quota.size.1=0x0000000000000000000000000000 > 00000000000000000001 > > > > [root at ml-ctl-ser01 sunyuyusun]$ salt "ml-storage-ser2*.nmg01" cmd.run > "getfattr -d -m. -e hex /data4/bricks/project2/371_37829/test-dir" > ml-storage-ser25.nmg01: > getfattr: Removing leading '/' from absolute path names > # file: data4/bricks/project2/371_37829/test-dir > trusted.gfid=0x57e0a8945a224ab4be2c6c71eada6217 > trusted.glusterfs.dht=0xdbde0bb10000000024924924279e79e6 > trusted.glusterfs.quota.dirty=0x3000 > trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1> 0x000000000000000000000000000000000000000000000001 > trusted.glusterfs.quota.size.1=0x0000000000000000000000000000 > 00000000000000000001 > ml-storage-ser27.nmg01: > getfattr: Removing leading '/' from absolute path names > # file: data4/bricks/project2/371_37829/test-dir > trusted.gfid=0x57e0a8945a224ab4be2c6c71eada6217 > trusted.glusterfs.dht=0xdbde0bb100000000279e79e72aaaaaa9 > trusted.glusterfs.quota.dirty=0x3000 > trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1> 0x000000000000000000000000000000000000000000000001 > trusted.glusterfs.quota.size.1=0x0000000000000000000000000000 > 00000000000000000001 > ml-storage-ser28.nmg01: > getfattr: Removing leading '/' from absolute path names > # file: data4/bricks/project2/371_37829/test-dir > trusted.gfid=0x57e0a8945a224ab4be2c6c71eada6217 > trusted.glusterfs.dht=0xdbde0bb1000000002aaaaaaa2db6db6c > trusted.glusterfs.quota.dirty=0x3000 > trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1> 0x000000000000000000000000000000000000000000000001 > trusted.glusterfs.quota.size.1=0x0000000000000000000000000000 > 00000000000000000001 > ml-storage-ser23.nmg01: > getfattr: Removing leading '/' from absolute path names > # file: data4/bricks/project2/371_37829/test-dir > trusted.gfid=0x57e0a8945a224ab4be2c6c71eada6217 > trusted.glusterfs.dht=0xdbde0bb1000000002186186124924923 > trusted.glusterfs.quota.dirty=0x3000 > trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1> 0x000000000000000000000000000000000000000000000001 > trusted.glusterfs.quota.size.1=0x0000000000000000000000000000 > 00000000000000000001 > ml-storage-ser26.nmg01: > getfattr: Removing leading '/' from absolute path names > # file: data4/bricks/project2/371_37829/test-dir > trusted.gfid=0x57e0a8945a224ab4be2c6c71eada6217 > trusted.glusterfs.dht=0xdbde0bb100000000279e79e72aaaaaa9 > trusted.glusterfs.quota.dirty=0x3000 > trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1> 0x000000000000000000000000000000000000000000000001 > trusted.glusterfs.quota.size.1=0x0000000000000000000000000000 > 00000000000000000001 > ml-storage-ser29.nmg01: > getfattr: Removing leading '/' from absolute path names > # file: data4/bricks/project2/371_37829/test-dir > trusted.gfid=0x57e0a8945a224ab4be2c6c71eada6217 > trusted.glusterfs.dht=0xdbde0bb1000000002aaaaaaa2db6db6c > trusted.glusterfs.quota.dirty=0x3000 > trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1> 0x000000000000000000000000000000000000000000000001 > trusted.glusterfs.quota.size.1=0x0000000000000000000000000000 > 00000000000000000001 > ml-storage-ser22.nmg01: > getfattr: Removing leading '/' from absolute path names > # file: data4/bricks/project2/371_37829/test-dir > trusted.gfid=0x57e0a8945a224ab4be2c6c71eada6217 > trusted.glusterfs.dht=0xdbde0bb1000000002186186124924923 > trusted.glusterfs.quota.dirty=0x3000 > trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1> 0x000000000000000000000000000000000000000000000001 > trusted.glusterfs.quota.size.1=0x0000000000000000000000000000 > 00000000000000000001 > ml-storage-ser24.nmg01: > getfattr: Removing leading '/' from absolute path names > # file: data4/bricks/project2/371_37829/test-dir > trusted.gfid=0x57e0a8945a224ab4be2c6c71eada6217 > trusted.glusterfs.dht=0xdbde0bb10000000024924924279e79e6 > trusted.glusterfs.quota.dirty=0x3000 > trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1> 0x000000000000000000000000000000000000000000000001 > trusted.glusterfs.quota.size.1=0x0000000000000000000000000000 > 00000000000000000001 > ml-storage-ser20.nmg01: > getfattr: Removing leading '/' from absolute path names > # file: data4/bricks/project2/371_37829/test-dir > c=0x57e0a8945a224ab4be2c6c71eada6217 > trusted.glusterfs.dht=0xdbde0bb1000000001e79e79e21861860 > trusted.glusterfs.quota.dirty=0x3000 > trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1> 0x000000000000000000000000000000000000000000000001 > trusted.glusterfs.quota.size.1=0x0000000000000000000000000000 > 00000000000000000001 > ml-storage-ser21.nmg01: > getfattr: Removing leading '/' from absolute path names > # file: data4/bricks/project2/371_37829/test-dir > trusted.gfid=0x57e0a8945a224ab4be2c6c71eada6217 > trusted.glusterfs.dht=0xdbde0bb1000000001e79e79e21861860 > trusted.glusterfs.quota.dirty=0x3000 > trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1> 0x000000000000000000000000000000000000000000000001 > trusted.glusterfs.quota.size.1=0x0000000000000000000000000000 > 00000000000000000001 > [root at ml-ctl-ser01 sunyuyusun]$ > > Is the infomation listed above what you need? I can see the two directory > (src and dst) are of diffenent trusted.gfid. > > also I have another test and basically certain that when i enable quota, > this will happen in one mount point(i only mount the test volume in one > mount point) > > > > Best Regards > Yours > Yu > > > > Raghavendra Gowdappa <rgowdapp at redhat.com> ?2018?9?5??? ??12:40??? > >> >> >> On Wed, Sep 5, 2018 at 9:21 AM, yu sun <sunyu1949 at gmail.com> wrote: >> >>> thank you , I will investigate and have a try. >>> >> >> Please get all the extended attributes of src and dst from all bricks. >> That should help us to (dis)prove whether its same bug. >> >> >>> BYW, I found another thing: >>> 1. I found this only occur when I try to mv dir to a subdir. >>> 2. I create a similar volume with default option and without quota >>> enabled, i found its ok, mv seems no problem. >>> >>> so I think it's possibly the volume option led to this problem, but I >>> dont know which options and how to resolve this, the volume have about 25T >>> data. >>> >>> Best Regards. >>> >>> Raghavendra Gowdappa <rgowdapp at redhat.com> ?2018?9?5??? ??10:50??? >>> >>>> >>>> >>>> On Tue, Sep 4, 2018 at 5:28 PM, yu sun <sunyu1949 at gmail.com> wrote: >>>> >>>>> Hi all: >>>>> >>>>> I have a replicated volume project2 with info: >>>>> Volume Name: project2 Type: Distributed-Replicate Volume ID: >>>>> 60175b8e-de0e-4409-81ae-7bb5eb5cacbf Status: Started Snapshot Count: >>>>> 0 Number of Bricks: 84 x 2 = 168 Transport-type: tcp Bricks: Brick1: >>>>> node20:/data2/bricks/project2 Brick2: node21:/data2/bricks/project2 Brick3: >>>>> node22:/data2/bricks/project2 Brick4: node23:/data2/bricks/project2 Brick5: >>>>> node24:/data2/bricks/project2 Brick6: node25:/data2/bricks/project2 Brick7: >>>>> node26:/data2/bricks/project2 Brick8: node27:/data2/bricks/project2 Brick9: >>>>> node28:/data2/bricks/project2 Brick10: node29:/data2/bricks/project2 >>>>> Brick11: node30:/data2/bricks/project2 Brick12: >>>>> node31:/data2/bricks/project2 Brick13: node32:/data2/bricks/project2 >>>>> Brick14: node33:/data2/bricks/project2 Brick15: >>>>> node20:/data3/bricks/project2 Brick16: node21:/data3/bricks/project2 >>>>> Brick17: node22:/data3/bricks/project2 Brick18: >>>>> node23:/data3/bricks/project2 Brick19: node24:/data3/bricks/project2 >>>>> Brick20: node25:/data3/bricks/project2 Brick21: >>>>> node26:/data3/bricks/project2 Brick22: node27:/data3/bricks/project2 >>>>> Brick23: node28:/data3/bricks/project2 Brick24: >>>>> node29:/data3/bricks/project2 Brick25: node30:/data3/bricks/project2 >>>>> Brick26: node31:/data3/bricks/project2 Brick27: >>>>> node32:/data3/bricks/project2 Brick28: node33:/data3/bricks/project2 >>>>> Brick29: node20:/data4/bricks/project2 Brick30: >>>>> node21:/data4/bricks/project2 Brick31: node22:/data4/bricks/project2 >>>>> Brick32: node23:/data4/bricks/project2 Brick33: >>>>> node24:/data4/bricks/project2 Brick34: node25:/data4/bricks/project2 >>>>> Brick35: node26:/data4/bricks/project2 Brick36: >>>>> node27:/data4/bricks/project2 Brick37: node28:/data4/bricks/project2 >>>>> Brick38: node29:/data4/bricks/project2 Brick39: >>>>> node30:/data4/bricks/project2 Brick40: node31:/data4/bricks/project2 >>>>> Brick41: node32:/data4/bricks/project2 Brick42: >>>>> node33:/data4/bricks/project2 Brick43: node20:/data5/bricks/project2 >>>>> Brick44: node21:/data5/bricks/project2 Brick45: >>>>> node22:/data5/bricks/project2 Brick46: node23:/data5/bricks/project2 >>>>> Brick47: node24:/data5/bricks/project2 Brick48: >>>>> node25:/data5/bricks/project2 Brick49: node26:/data5/bricks/project2 >>>>> Brick50: node27:/data5/bricks/project2 Brick51: >>>>> node28:/data5/bricks/project2 Brick52: node29:/data5/bricks/project2 >>>>> Brick53: node30:/data5/bricks/project2 Brick54: >>>>> node31:/data5/bricks/project2 Brick55: node32:/data5/bricks/project2 >>>>> Brick56: node33:/data5/bricks/project2 Brick57: >>>>> node20:/data6/bricks/project2 Brick58: node21:/data6/bricks/project2 >>>>> Brick59: node22:/data6/bricks/project2 Brick60: >>>>> node23:/data6/bricks/project2 Brick61: node24:/data6/bricks/project2 >>>>> Brick62: node25:/data6/bricks/project2 Brick63: >>>>> node26:/data6/bricks/project2 Brick64: node27:/data6/bricks/project2 >>>>> Brick65: node28:/data6/bricks/project2 Brick66: >>>>> node29:/data6/bricks/project2 Brick67: node30:/data6/bricks/project2 >>>>> Brick68: node31:/data6/bricks/project2 Brick69: >>>>> node32:/data6/bricks/project2 Brick70: node33:/data6/bricks/project2 >>>>> Brick71: node20:/data7/bricks/project2 Brick72: >>>>> node21:/data7/bricks/project2 Brick73: node22:/data7/bricks/project2 >>>>> Brick74: node23:/data7/bricks/project2 Brick75: >>>>> node24:/data7/bricks/project2 Brick76: node25:/data7/bricks/project2 >>>>> Brick77: node26:/data7/bricks/project2 Brick78: >>>>> node27:/data7/bricks/project2 Brick79: node28:/data7/bricks/project2 >>>>> Brick80: node29:/data7/bricks/project2 Brick81: >>>>> node30:/data7/bricks/project2 Brick82: node31:/data7/bricks/project2 >>>>> Brick83: node32:/data7/bricks/project2 Brick84: >>>>> node33:/data7/bricks/project2 Brick85: node20:/data8/bricks/project2 >>>>> Brick86: node21:/data8/bricks/project2 Brick87: >>>>> node22:/data8/bricks/project2 Brick88: node23:/data8/bricks/project2 >>>>> Brick89: node24:/data8/bricks/project2 Brick90: >>>>> node25:/data8/bricks/project2 Brick91: node26:/data8/bricks/project2 >>>>> Brick92: node27:/data8/bricks/project2 Brick93: >>>>> node28:/data8/bricks/project2 Brick94: node29:/data8/bricks/project2 >>>>> Brick95: node30:/data8/bricks/project2 Brick96: >>>>> node31:/data8/bricks/project2 Brick97: node32:/data8/bricks/project2 >>>>> Brick98: node33:/data8/bricks/project2 Brick99: >>>>> node20:/data9/bricks/project2 Brick100: node21:/data9/bricks/project2 >>>>> Brick101: node22:/data9/bricks/project2 Brick102: >>>>> node23:/data9/bricks/project2 Brick103: node24:/data9/bricks/project2 >>>>> Brick104: node25:/data9/bricks/project2 Brick105: >>>>> node26:/data9/bricks/project2 Brick106: node27:/data9/bricks/project2 >>>>> Brick107: node28:/data9/bricks/project2 Brick108: >>>>> node29:/data9/bricks/project2 Brick109: node30:/data9/bricks/project2 >>>>> Brick110: node31:/data9/bricks/project2 Brick111: >>>>> node32:/data9/bricks/project2 Brick112: node33:/data9/bricks/project2 >>>>> Brick113: node20:/data10/bricks/project2 Brick114: >>>>> node21:/data10/bricks/project2 Brick115: node22:/data10/bricks/project2 >>>>> Brick116: node23:/data10/bricks/project2 Brick117: >>>>> node24:/data10/bricks/project2 Brick118: node25:/data10/bricks/project2 >>>>> Brick119: node26:/data10/bricks/project2 Brick120: >>>>> node27:/data10/bricks/project2 Brick121: node28:/data10/bricks/project2 >>>>> Brick122: node29:/data10/bricks/project2 Brick123: >>>>> node30:/data10/bricks/project2 Brick124: node31:/data10/bricks/project2 >>>>> Brick125: node32:/data10/bricks/project2 Brick126: >>>>> node33:/data10/bricks/project2 Brick127: node20:/data11/bricks/project2 >>>>> Brick128: node21:/data11/bricks/project2 Brick129: >>>>> node22:/data11/bricks/project2 Brick130: node23:/data11/bricks/project2 >>>>> Brick131: node24:/data11/bricks/project2 Brick132: >>>>> node25:/data11/bricks/project2 Brick133: node26:/data11/bricks/project2 >>>>> Brick134: node27:/data11/bricks/project2 Brick135: >>>>> node28:/data11/bricks/project2 Brick136: node29:/data11/bricks/project2 >>>>> Brick137: node30:/data11/bricks/project2 Brick138: >>>>> node31:/data11/bricks/project2 Brick139: node32:/data11/bricks/project2 >>>>> Brick140: node33:/data11/bricks/project2 Brick141: >>>>> node20:/data12/bricks/project2 Brick142: node21:/data12/bricks/project2 >>>>> Brick143: node22:/data12/bricks/project2 Brick144: >>>>> node23:/data12/bricks/project2 Brick145: node24:/data12/bricks/project2 >>>>> Brick146: node25:/data12/bricks/project2 Brick147: >>>>> node26:/data12/bricks/project2 Brick148: node27:/data12/bricks/project2 >>>>> Brick149: node28:/data12/bricks/project2 Brick150: >>>>> node29:/data12/bricks/project2 Brick151: node30:/data12/bricks/project2 >>>>> Brick152: node31:/data12/bricks/project2 Brick153: >>>>> node32:/data12/bricks/project2 Brick154: node33:/data12/bricks/project2 >>>>> Brick155: node20:/data13/bricks/project2 Brick156: >>>>> node21:/data13/bricks/project2 Brick157: node22:/data13/bricks/project2 >>>>> Brick158: node23:/data13/bricks/project2 Brick159: >>>>> node24:/data13/bricks/project2 Brick160: node25:/data13/bricks/project2 >>>>> Brick161: node26:/data13/bricks/project2 Brick162: >>>>> node27:/data13/bricks/project2 Brick163: node28:/data13/bricks/project2 >>>>> Brick164: node29:/data13/bricks/project2 Brick165: >>>>> node30:/data13/bricks/project2 Brick166: node31:/data13/bricks/project2 >>>>> Brick167: node32:/data13/bricks/project2 Brick168: >>>>> node33:/data13/bricks/project2 Options Reconfigured: >>>>> performance.force-readdirp: on performance.write-behind: off >>>>> performance.stat-prefetch: on performance.client-io-threads: on >>>>> nfs.disable: on transport.address-family: inet features.quota: on >>>>> features.inode-quota: on features.quota-deem-statfs: on >>>>> cluster.readdir-optimize: on cluster.lookup-optimize: on >>>>> dht.force-readdirp: off client.event-threads: 10 server.event-threads: 10 >>>>> performance.readdir-ahead: on performance.io-cache: on >>>>> performance.flush-behind: on performance.cache-size: 5GB >>>>> performance.cache-max-file-size: 1MB performance.write-behind-window-size: >>>>> 10MB performance.read-ahead: off network.remote-dio: enable >>>>> performance.strict-o-direct: disable performance.io-thread-count: 25 >>>>> >>>>> >>>>> the volume looks ok, and I mount this volume on my client machine: >>>>> mount -t glusterfs -o oom-score-adj=-999 -o direct-io-mode=disable -o >>>>> use-readdirp=no node20:/project2 /mnt/project2 >>>>> >>>>> I have a directory in /mnt/project2/, but when I mv the directory to >>>>> other dirs, files in the dir lost while tree or ls the dir, some files >>>>> missing, my operations is list as below: >>>>> >>>> >>>> Looks very similar to: >>>> https://bugzilla.redhat.com/show_bug.cgi?id=1118762 >>>> https://bugzilla.redhat.com/show_bug.cgi?id=1337394 >>>> >>>> >>>> >>>>> root at ml-gpu-ser129.nmg01:/mnt/project2/371_37829$ mkdir test-dir >>>>> root at ml-gpu-ser129.nmg01:/mnt/project2/371_37829$ tree >>>>> . >>>>> ??? face_landmarks >>>>> ? ??? alive >>>>> ? ??? logs_20180823_28 >>>>> ? ??? info_000000.out >>>>> ? ??? info_000001.out >>>>> ? ??? info_000002.out >>>>> ? ??? info_000003.out >>>>> ? ??? info_000004.out >>>>> ? ??? info_000005.out >>>>> ? ??? info_000006.out >>>>> ? ??? info_000007.out >>>>> ? ??? info_000008.out >>>>> ? ??? info_000009.out >>>>> ? ??? info_000010.out >>>>> ? ??? info_000011.out >>>>> ? ??? info_000012.out >>>>> ? ??? info_000013.out >>>>> ? ??? info_000014.out >>>>> ? ??? info_000015.out >>>>> ? ??? info_000016.out >>>>> ? ??? info_000017.out >>>>> ? ??? info_000018.out >>>>> ? ??? info_000019.out >>>>> ??? test-dir >>>>> >>>>> 4 directories, 20 files >>>>> root at ml-gpu-ser129.nmg01:/mnt/project2/371_37829$ mv face_landmarks/ >>>>> test-dir/ >>>>> root at ml-gpu-ser129.nmg01:/mnt/project2/371_37829$ tree >>>>> . >>>>> ??? test-dir >>>>> ??? face_landmarks >>>>> >>>>> 2 directories, 0 files >>>>> root at ml-gpu-ser129.nmg01:/mnt/project2/371_37829$ cd >>>>> test-dir/face_landmarks/ >>>>> root at ml-gpu-ser129.nmg01:/mnt/project2/371_37829/test-dir/face_landmarks$ >>>>> ls >>>>> root at ml-gpu-ser129.nmg01:/mnt/project2/371_37829/test-dir/face_landmarks$ >>>>> cd .. >>>>> root at ml-gpu-ser129.nmg01:/mnt/project2/371_37829/test-dir$ mv >>>>> face_landmarks/ .. >>>>> root at ml-gpu-ser129.nmg01:/mnt/project2/371_37829/test-dir$ cd .. >>>>> root at ml-gpu-ser129.nmg01:/mnt/project2/371_37829$ tree >>>>> . >>>>> ??? face_landmarks >>>>> ? ??? alive >>>>> ??? test-dir >>>>> >>>>> 3 directories, 0 files >>>>> root at ml-gpu-ser129.nmg01:/mnt/project2/371_37829$ >>>>> >>>>> I think i make some mistakes with volume option, buti i am not sure, >>>>> so how can i find the lost files? the files seems still int the directory, >>>>> because i cant remove the directory and rm tell me "Not empty directory" >>>>> >>>> >>>> Its likely that src and dst of mv having same gfid and that's causing >>>> the issues. Can you look into both src and dst paths on all bricks? Union >>>> of contents of both directories should give all the files in the src >>>> directory before mv. Once found you can, >>>> * keep a backup of contents of src and dst on all bricks >>>> * remove trusted.gfid xattr on src and dst from all bricks >>>> * remove gfid handle (.glusterfs/<first two characters of gfid>/<second >>>> set of two characters of gfid>/<gfid> on each brick) >>>> * disable readdirplus in entire stack (maybe you can use a tmp mount >>>> for this) [1] >>>> * stat src and dst on a mount point with readdirplus disabled. >>>> * Now you'll see two directories src and dst on mountpoint. You can >>>> copy the contents of both into a new directory >>>> >>>> [1] https://lists.gluster.org/pipermail/gluster-users/2017- >>>> March/030148.html >>>> >>>> >>>>> >>>>> Any suggestions is appreciated. >>>>> Many Thanks >>>>> >>>>> Best regards >>>>> Yu >>>>> >>>>> _______________________________________________ >>>>> Gluster-users mailing list >>>>> Gluster-users at gluster.org >>>>> https://lists.gluster.org/mailman/listinfo/gluster-users >>>>> >>>> >>>> >> > _______________________________________________ > Gluster-users mailing list > Gluster-users at gluster.org > https://lists.gluster.org/mailman/listinfo/gluster-users >-------------- next part -------------- An HTML attachment was scrubbed... URL: <http://lists.gluster.org/pipermail/gluster-users/attachments/20180905/29268d26/attachment.html>
On Wed, Sep 5, 2018 at 12:46 PM, yu sun <sunyu1949 at gmail.com> wrote:> > [root at ml-ctl-ser01 sunyuyusun]$ salt "ml-storage-ser2*.nmg01" cmd.run > "getfattr -d -m. -e hex /data2/bricks/project2/371_37829/face_landmarks" > ml-storage-ser24.nmg01: > getfattr: Removing leading '/' from absolute path names > # file: data2/bricks/project2/371_37829/face_landmarks > trusted.gfid=0x3764e261741d423d8c8eb6ad5b716429 > trusted.glusterfs.dht=0xdbde0bb100000000b0c30c2eb3cf3cf0 > trusted.glusterfs.quota.dirty=0x3000 > trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1> 0x00000000000d760000000000000000010000000000000003 > trusted.glusterfs.quota.size.1=0x00000000000d7600000000000000 > 00010000000000000003 > ml-storage-ser26.nmg01: > getfattr: Removing leading '/' from absolute path names > # file: data2/bricks/project2/371_37829/face_landmarks > trusted.gfid=0x3764e261741d423d8c8eb6ad5b716429 > trusted.glusterfs.dht=0xdbde0bb100000000d249248fd5555551 > trusted.glusterfs.quota.dirty=0x3000 > trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1> 0x000000000000000000000000000000000000000000000003 > trusted.glusterfs.quota.size.1=0x0000000000000000000000000000 > 00000000000000000003 > ml-storage-ser23.nmg01: > getfattr: Removing leading '/' from absolute path names > # file: data2/bricks/project2/371_37829/face_landmarks > trusted.gfid=0x3764e261741d423d8c8eb6ad5b716429 > trusted.glusterfs.dht=0xdbde0bb1000000008f3cf3cd9249248f > trusted.glusterfs.quota.dirty=0x3000 > trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1> 0x000000000000000000000000000000000000000000000003 > trusted.glusterfs.quota.size.1=0x0000000000000000000000000000 > 00000000000000000003 > ml-storage-ser27.nmg01: > getfattr: Removing leading '/' from absolute path names > # file: data2/bricks/project2/371_37829/face_landmarks > trusted.gfid=0x3764e261741d423d8c8eb6ad5b716429 > trusted.glusterfs.dht=0xdbde0bb100000000d249248fd5555551 > trusted.glusterfs.quota.dirty=0x3000 > trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1> 0x000000000000000000000000000000000000000000000003 > trusted.glusterfs.quota.size.1=0x0000000000000000000000000000 > 00000000000000000003 > ml-storage-ser25.nmg01: > getfattr: Removing leading '/' from absolute path names > # file: data2/bricks/project2/371_37829/face_landmarks > trusted.gfid=0x3764e261741d423d8c8eb6ad5b716429 > trusted.glusterfs.dht=0xdbde0bb100000000b0c30c2eb3cf3cf0 > trusted.glusterfs.quota.dirty=0x3000 > trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1> 0x00000000000d760000000000000000010000000000000003 > trusted.glusterfs.quota.size.1=0x00000000000d7600000000000000 > 00010000000000000003 > ml-storage-ser28.nmg01: > getfattr: Removing leading '/' from absolute path names > # file: data2/bricks/project2/371_37829/face_landmarks > trusted.gfid=0x3764e261741d423d8c8eb6ad5b716429 > trusted.glusterfs.dht=0xdbde0bb100000000f3cf3cf0f6db6db2 > trusted.glusterfs.quota.dirty=0x3000 > trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1> 0x000000000000000000000000000000000000000000000003 > trusted.glusterfs.quota.size.1=0x0000000000000000000000000000 > 00000000000000000003 > ml-storage-ser29.nmg01: > getfattr: Removing leading '/' from absolute path names > # file: data2/bricks/project2/371_37829/face_landmarks > trusted.gfid=0x3764e261741d423d8c8eb6ad5b716429 > trusted.glusterfs.dht=0xdbde0bb100000000f3cf3cf0f6db6db2 > trusted.glusterfs.quota.dirty=0x3000 > trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1> 0x000000000000000000000000000000000000000000000003 > trusted.glusterfs.quota.size.1=0x0000000000000000000000000000 > 00000000000000000003 > ml-storage-ser20.nmg01: > getfattr: Removing leading '/' from absolute path names > # file: data2/bricks/project2/371_37829/face_landmarks > trusted.gfid=0x3764e261741d423d8c8eb6ad5b716429 > trusted.glusterfs.dht=0xdbde0bb1000000008c30c30a8f3cf3cc > trusted.glusterfs.quota.dirty=0x3000 > trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1> 0x000000000000000000000000000000000000000000000003 > trusted.glusterfs.quota.size.1=0x0000000000000000000000000000 > 00000000000000000003 > ml-storage-ser22.nmg01: > getfattr: Removing leading '/' from absolute path names > # file: data2/bricks/project2/371_37829/face_landmarks > trusted.gfid=0x3764e261741d423d8c8eb6ad5b716429 > trusted.glusterfs.dht=0xdbde0bb1000000008f3cf3cd9249248f > trusted.glusterfs.quota.dirty=0x3000 > trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1> 0x000000000000000000000000000000000000000000000003 > trusted.glusterfs.quota.size.1=0x0000000000000000000000000000 > 00000000000000000003 > ml-storage-ser21.nmg01: > getfattr: Removing leading '/' from absolute path names > # file: data2/bricks/project2/371_37829/face_landmarks > trusted.gfid=0x3764e261741d423d8c8eb6ad5b716429 > trusted.glusterfs.dht=0xdbde0bb1000000008c30c30a8f3cf3cc > trusted.glusterfs.quota.dirty=0x3000 > trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1> 0x000000000000000000000000000000000000000000000003 > trusted.glusterfs.quota.size.1=0x0000000000000000000000000000 > 00000000000000000003 > [root at ml-ctl-ser01 sunyuyusun]$ salt "ml-storage-ser2*.nmg01" cmd.run > "getfattr -d -m. -e hex /data3/bricks/project2/371_37829/face_landmarks" > ml-storage-ser28.nmg01: > getfattr: Removing leading '/' from absolute path names > # file: data3/bricks/project2/371_37829/face_landmarks > trusted.gfid=0x3764e261741d423d8c8eb6ad5b716429 > trusted.glusterfs.dht=0xdbde0bb1000000009555555398618615 > trusted.glusterfs.quota.dirty=0x3000 > trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1> 0x00000000001aec0000000000000000020000000000000003 > trusted.glusterfs.quota.size.1=0x00000000001aec00000000000000 > 00020000000000000003 > ml-storage-ser25.nmg01: > getfattr: Removing leading '/' from absolute path names > # file: data3/bricks/project2/371_37829/face_landmarks > trusted.gfid=0x3764e261741d423d8c8eb6ad5b716429 > trusted.glusterfs.dht=0xdbde0bb100000000892492478c30c309 > trusted.glusterfs.quota.dirty=0x3000 > trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1> 0x000000000000000000000000000000000000000000000003 > trusted.glusterfs.quota.size.1=0x0000000000000000000000000000 > 00000000000000000003 > ml-storage-ser24.nmg01: > getfattr: Removing leading '/' from absolute path names > # file: data3/bricks/project2/371_37829/face_landmarks > trusted.gfid=0x3764e261741d423d8c8eb6ad5b716429 > trusted.glusterfs.dht=0xdbde0bb100000000892492478c30c309 > trusted.glusterfs.quota.dirty=0x3000 > trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1> 0x000000000000000000000000000000000000000000000003 > trusted.glusterfs.quota.size.1=0x0000000000000000000000000000 > 00000000000000000003 > ml-storage-ser27.nmg01: > getfattr: Removing leading '/' from absolute path names > # file: data3/bricks/project2/371_37829/face_landmarks > trusted.gfid=0x3764e261741d423d8c8eb6ad5b716429 > trusted.glusterfs.dht=0xdbde0bb1000000009249249095555552 > trusted.glusterfs.quota.dirty=0x3000 > trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1> 0x000000000000000000000000000000000000000000000003 > trusted.glusterfs.quota.size.1=0x0000000000000000000000000000 > 00000000000000000003 > ml-storage-ser23.nmg01: > getfattr: Removing leading '/' from absolute path names > # file: data3/bricks/project2/371_37829/face_landmarks > trusted.gfid=0x3764e261741d423d8c8eb6ad5b716429 > trusted.glusterfs.dht=0xdbde0bb10000000079e79e787cf3cf3a > trusted.glusterfs.quota.dirty=0x3000 > trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1> 0x000000000000000000000000000000000000000000000003 > trusted.glusterfs.quota.size.1=0x0000000000000000000000000000 > 00000000000000000003 > ml-storage-ser22.nmg01: > getfattr: Removing leading '/' from absolute path names > # file: data3/bricks/project2/371_37829/face_landmarks > trusted.gfid=0x3764e261741d423d8c8eb6ad5b716429 > trusted.glusterfs.dht=0xdbde0bb10000000079e79e787cf3cf3a > trusted.glusterfs.quota.dirty=0x3000 > trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1> 0x000000000000000000000000000000000000000000000003 > trusted.glusterfs.quota.size.1=0x0000000000000000000000000000 > 00000000000000000003 > ml-storage-ser29.nmg01: > getfattr: Removing leading '/' from absolute path names > # file: data3/bricks/project2/371_37829/face_landmarks > trusted.gfid=0x3764e261741d423d8c8eb6ad5b716429 > trusted.glusterfs.dht=0xdbde0bb1000000009555555398618615 > trusted.glusterfs.quota.dirty=0x3000 > trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1> 0x00000000001aec0000000000000000020000000000000003 > trusted.glusterfs.quota.size.1=0x00000000001aec00000000000000 > 00020000000000000003 > ml-storage-ser26.nmg01: > getfattr: Removing leading '/' from absolute path names > # file: data3/bricks/project2/371_37829/face_landmarks > trusted.gfid=0x3764e261741d423d8c8eb6ad5b716429 > trusted.glusterfs.dht=0xdbde0bb1000000009249249095555552 > trusted.glusterfs.quota.dirty=0x3000 > trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1> 0x000000000000000000000000000000000000000000000003 > trusted.glusterfs.quota.size.1=0x0000000000000000000000000000 > 00000000000000000003 > ml-storage-ser21.nmg01: > getfattr: Removing leading '/' from absolute path names > # file: data3/bricks/project2/371_37829/face_landmarks > trusted.gfid=0x3764e261741d423d8c8eb6ad5b716429 > trusted.glusterfs.dht=0xdbde0bb100000000586186175b6db6d9 > trusted.glusterfs.quota.dirty=0x3000 > trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1> 0x000000000000000000000000000000000000000000000003 > trusted.glusterfs.quota.size.1=0x0000000000000000000000000000 > 00000000000000000003 > ml-storage-ser20.nmg01: > getfattr: Removing leading '/' from absolute path names > # file: data3/bricks/project2/371_37829/face_landmarks > trusted.gfid=0x3764e261741d423d8c8eb6ad5b716429 > trusted.glusterfs.dht=0xdbde0bb100000000586186175b6db6d9 > trusted.glusterfs.quota.dirty=0x3000 > trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1> 0x000000000000000000000000000000000000000000000003 > trusted.glusterfs.quota.size.1=0x0000000000000000000000000000 > 00000000000000000003 > > > > [root at ml-ctl-ser01 sunyuyusun]$ salt "ml-storage-ser2*.nmg01" cmd.run > "getfattr -d -m. -e hex /data4/bricks/project2/371_37829/face_landmarks" > ml-storage-ser25.nmg01: > getfattr: Removing leading '/' from absolute path names > # file: data4/bricks/project2/371_37829/face_landmarks > trusted.gfid=0x3764e261741d423d8c8eb6ad5b716429 > trusted.glusterfs.dht=0xdbde0bb100000000a4924922a79e79e4 > trusted.glusterfs.quota.dirty=0x3000 > trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1> 0x000000000000000000000000000000000000000000000003 > trusted.glusterfs.quota.size.1=0x0000000000000000000000000000 > 00000000000000000003 > ml-storage-ser23.nmg01: > getfattr: Removing leading '/' from absolute path names > # file: data4/bricks/project2/371_37829/face_landmarks > trusted.gfid=0x3764e261741d423d8c8eb6ad5b716429 > trusted.glusterfs.dht=0xdbde0bb100000000a186185fa4924921 > trusted.glusterfs.quota.dirty=0x3000 > trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1> 0x000000000000000000000000000000000000000000000003 > trusted.glusterfs.quota.size.1=0x0000000000000000000000000000 > 00000000000000000003 > ml-storage-ser24.nmg01: > getfattr: Removing leading '/' from absolute path names > # file: data4/bricks/project2/371_37829/face_landmarks > trusted.gfid=0x3764e261741d423d8c8eb6ad5b716429 > trusted.glusterfs.dht=0xdbde0bb100000000a4924922a79e79e4 > trusted.glusterfs.quota.dirty=0x3000 > trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1> 0x000000000000000000000000000000000000000000000003 > trusted.glusterfs.quota.size.1=0x0000000000000000000000000000 > 00000000000000000003 > ml-storage-ser28.nmg01: > getfattr: Removing leading '/' from absolute path names > # file: data4/bricks/project2/371_37829/face_landmarks > trusted.gfid=0x3764e261741d423d8c8eb6ad5b716429 > trusted.glusterfs.dht=0xdbde0bb100000000aaaaaaa8adb6db6a > trusted.glusterfs.quota.dirty=0x3000 > trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1> 0x000000000000000000000000000000000000000000000003 > trusted.glusterfs.quota.size.1=0x0000000000000000000000000000 > 00000000000000000003 > ml-storage-ser27.nmg01: > getfattr: Removing leading '/' from absolute path names > # file: data4/bricks/project2/371_37829/face_landmarks > trusted.gfid=0x3764e261741d423d8c8eb6ad5b716429 > trusted.glusterfs.dht=0xdbde0bb100000000a79e79e5aaaaaaa7 > trusted.glusterfs.quota.dirty=0x3000 > trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1> 0x000000000000000000000000000000000000000000000003 > trusted.glusterfs.quota.size.1=0x0000000000000000000000000000 > 00000000000000000003 > ml-storage-ser26.nmg01: > getfattr: Removing leading '/' from absolute path names > # file: data4/bricks/project2/371_37829/face_landmarks > trusted.gfid=0x3764e261741d423d8c8eb6ad5b716429 > trusted.glusterfs.dht=0xdbde0bb100000000a79e79e5aaaaaaa7 > trusted.glusterfs.quota.dirty=0x3000 > trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1> 0x000000000000000000000000000000000000000000000003 > trusted.glusterfs.quota.size.1=0x0000000000000000000000000000 > 00000000000000000003 > ml-storage-ser21.nmg01: > getfattr: Removing leading '/' from absolute path names > # file: data4/bricks/project2/371_37829/face_landmarks > trusted.gfid=0x3764e261741d423d8c8eb6ad5b716429 > trusted.glusterfs.dht=0xdbde0bb1000000009e79e79ca186185e > trusted.glusterfs.quota.dirty=0x3000 > trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1> 0x000000000000000000000000000000000000000000000003 > trusted.glusterfs.quota.size.1=0x0000000000000000000000000000 > 00000000000000000003 > ml-storage-ser22.nmg01: > getfattr: Removing leading '/' from absolute path names > # file: data4/bricks/project2/371_37829/face_landmarks > trusted.gfid=0x3764e261741d423d8c8eb6ad5b716429 > trusted.glusterfs.dht=0xdbde0bb100000000a186185fa4924921 > trusted.glusterfs.quota.dirty=0x3000 > trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1> 0x000000000000000000000000000000000000000000000003 > trusted.glusterfs.quota.size.1=0x0000000000000000000000000000 > 00000000000000000003 > ml-storage-ser29.nmg01: > getfattr: Removing leading '/' from absolute path names > # file: data4/bricks/project2/371_37829/face_landmarks > trusted.gfid=0x3764e261741d423d8c8eb6ad5b716429 > trusted.glusterfs.dht=0xdbde0bb100000000aaaaaaa8adb6db6a > trusted.glusterfs.quota.dirty=0x3000 > trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1> 0x000000000000000000000000000000000000000000000003 > trusted.glusterfs.quota.size.1=0x0000000000000000000000000000 > 00000000000000000003 > ml-storage-ser20.nmg01: > getfattr: Removing leading '/' from absolute path names > # file: data4/bricks/project2/371_37829/face_landmarks > trusted.gfid=0x3764e261741d423d8c8eb6ad5b716429 > trusted.glusterfs.dht=0xdbde0bb1000000009e79e79ca186185e > trusted.glusterfs.quota.dirty=0x3000 > trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1> 0x000000000000000000000000000000000000000000000003 > trusted.glusterfs.quota.size.1=0x0000000000000000000000000000 > 00000000000000000003 > > > > > [root at ml-ctl-ser01 sunyuyusun]$ salt "ml-storage-ser2*.nmg01" cmd.run > "getfattr -d -m. -e hex /data2/bricks/project2/371_37829/test-dir" > ml-storage-ser23.nmg01: > getfattr: Removing leading '/' from absolute path names > # file: data2/bricks/project2/371_37829/test-dir > trusted.gfid=0x57e0a8945a224ab4be2c6c71eada6217 > trusted.glusterfs.dht=0xdbde0bb1000000000f3cf3cf12492491 > trusted.glusterfs.quota.dirty=0x3000 > trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1> 0x000000000000000000000000000000000000000000000001 > trusted.glusterfs.quota.size.1=0x0000000000000000000000000000 > 00000000000000000001 > ml-storage-ser24.nmg01: > getfattr: Removing leading '/' from absolute path names > # file: data2/bricks/project2/371_37829/test-dir > trusted.gfid=0x57e0a8945a224ab4be2c6c71eada6217 > trusted.glusterfs.dht=0xdbde0bb10000000030c30c3033cf3cf2 > trusted.glusterfs.quota.dirty=0x3000 > trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1> 0x000000000000000000000000000000000000000000000001 > trusted.glusterfs.quota.size.1=0x0000000000000000000000000000 > 00000000000000000001 > ml-storage-ser28.nmg01: > getfattr: Removing leading '/' from absolute path names > # file: data2/bricks/project2/371_37829/test-dir > trusted.gfid=0x57e0a8945a224ab4be2c6c71eada6217 > trusted.glusterfs.dht=0xdbde0bb10000000073cf3cf276db6db4 > trusted.glusterfs.quota.dirty=0x3000 > trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1> 0x000000000000000000000000000000000000000000000001 > trusted.glusterfs.quota.size.1=0x0000000000000000000000000000 > 00000000000000000001 > ml-storage-ser25.nmg01: > getfattr: Removing leading '/' from absolute path names > # file: data2/bricks/project2/371_37829/test-dir > trusted.gfid=0x57e0a8945a224ab4be2c6c71eada6217 > trusted.glusterfs.dht=0xdbde0bb10000000030c30c3033cf3cf2 > trusted.glusterfs.quota.dirty=0x3000 > trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1> 0x000000000000000000000000000000000000000000000001 > trusted.glusterfs.quota.size.1=0x0000000000000000000000000000 > 00000000000000000001 > ml-storage-ser26.nmg01: > getfattr: Removing leading '/' from absolute path names > # file: data2/bricks/project2/371_37829/test-dir > trusted.gfid=0x57e0a8945a224ab4be2c6c71eada6217 > trusted.glusterfs.dht=0xdbde0bb1000000005249249155555553 > trusted.glusterfs.quota.dirty=0x3000 > trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1> 0x000000000000000000000000000000000000000000000001 > trusted.glusterfs.quota.size.1=0x0000000000000000000000000000 > 00000000000000000001 > ml-storage-ser27.nmg01: > getfattr: Removing leading '/' from absolute path names > # file: data2/bricks/project2/371_37829/test-dir > trusted.gfid=0x57e0a8945a224ab4be2c6c71eada6217 > trusted.glusterfs.dht=0xdbde0bb1000000005249249155555553 > trusted.glusterfs.quota.dirty=0x3000 > trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1> 0x000000000000000000000000000000000000000000000001 > trusted.glusterfs.quota.size.1=0x0000000000000000000000000000 > 00000000000000000001 > ml-storage-ser22.nmg01: > getfattr: Removing leading '/' from absolute path names > # file: data2/bricks/project2/371_37829/test-dir > trusted.gfid=0x57e0a8945a224ab4be2c6c71eada6217 > trusted.glusterfs.dht=0xdbde0bb1000000000f3cf3cf12492491 > trusted.glusterfs.quota.dirty=0x3000 > trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1> 0x000000000000000000000000000000000000000000000001 > trusted.glusterfs.quota.size.1=0x0000000000000000000000000000 > 00000000000000000001 > ml-storage-ser29.nmg01: > getfattr: Removing leading '/' from absolute path names > # file: data2/bricks/project2/371_37829/test-dir > trusted.gfid=0x57e0a8945a224ab4be2c6c71eada6217 > trusted.glusterfs.dht=0xdbde0bb10000000073cf3cf276db6db4 > trusted.glusterfs.quota.dirty=0x3000 > trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1> 0x000000000000000000000000000000000000000000000001 > trusted.glusterfs.quota.size.1=0x0000000000000000000000000000 > 00000000000000000001 > ml-storage-ser21.nmg01: > getfattr: Removing leading '/' from absolute path names > # file: data2/bricks/project2/371_37829/test-dir > trusted.gfid=0x57e0a8945a224ab4be2c6c71eada6217 > trusted.glusterfs.dht=0xdbde0bb1000000000c30c30c0f3cf3ce > trusted.glusterfs.quota.dirty=0x3000 > trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1> 0x000000000000000000000000000000000000000000000001 > trusted.glusterfs.quota.size.1=0x0000000000000000000000000000 > 00000000000000000001 > ml-storage-ser20.nmg01: > getfattr: Removing leading '/' from absolute path names > # file: data2/bricks/project2/371_37829/test-dir > trusted.gfid=0x57e0a8945a224ab4be2c6c71eada6217 > trusted.glusterfs.dht=0xdbde0bb1000000000c30c30c0f3cf3ce > trusted.glusterfs.quota.dirty=0x3000 > trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1> 0x000000000000000000000000000000000000000000000001 > trusted.glusterfs.quota.size.1=0x0000000000000000000000000000 > 00000000000000000001 > > > > [root at ml-ctl-ser01 sunyuyusun]$ salt "ml-storage-ser2*.nmg01" cmd.run > "getfattr -d -m. -e hex /data3/bricks/project2/371_37829/test-dir" > ml-storage-ser23.nmg01: > getfattr: Removing leading '/' from absolute path names > # file: data3/bricks/project2/371_37829/test-dir > trusted.gfid=0x57e0a8945a224ab4be2c6c71eada6217 > trusted.glusterfs.dht=0xdbde0bb100000000f9e79e76fcf3cf38 > trusted.glusterfs.quota.dirty=0x3000 > trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1> 0x000000000000000000000000000000000000000000000001 > trusted.glusterfs.quota.size.1=0x0000000000000000000000000000 > 00000000000000000001 > ml-storage-ser26.nmg01: > getfattr: Removing leading '/' from absolute path names > # file: data3/bricks/project2/371_37829/test-dir > trusted.gfid=0x57e0a8945a224ab4be2c6c71eada6217 > trusted.glusterfs.dht=0xdbde0bb1000000001249249215555554 > trusted.glusterfs.quota.dirty=0x3000 > trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1> 0x000000000000000000000000000000000000000000000001 > trusted.glusterfs.quota.size.1=0x0000000000000000000000000000 > 00000000000000000001 > ml-storage-ser25.nmg01: > getfattr: Removing leading '/' from absolute path names > # file: data3/bricks/project2/371_37829/test-dir > trusted.gfid=0x57e0a8945a224ab4be2c6c71eada6217 > trusted.glusterfs.dht=0xdbde0bb100000000092492490c30c30b > trusted.glusterfs.quota.dirty=0x3000 > trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1> 0x000000000000000000000000000000000000000000000001 > trusted.glusterfs.quota.size.1=0x0000000000000000000000000000 > 00000000000000000001 > ml-storage-ser28.nmg01: > getfattr: Removing leading '/' from absolute path names > # file: data3/bricks/project2/371_37829/test-dir > trusted.gfid=0x57e0a8945a224ab4be2c6c71eada6217 > trusted.glusterfs.dht=0xdbde0bb1000000001555555518618617 > trusted.glusterfs.quota.dirty=0x3000 > trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1> 0x000000000000000000000000000000000000000000000001 > trusted.glusterfs.quota.size.1=0x0000000000000000000000000000 > 00000000000000000001 > ml-storage-ser24.nmg01: > getfattr: Removing leading '/' from absolute path names > # file: data3/bricks/project2/371_37829/test-dir > trusted.gfid=0x57e0a8945a224ab4be2c6c71eada6217 > trusted.glusterfs.dht=0xdbde0bb100000000092492490c30c30b > trusted.glusterfs.quota.dirty=0x3000 > trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1> 0x000000000000000000000000000000000000000000000001 > trusted.glusterfs.quota.size.1=0x0000000000000000000000000000 > 00000000000000000001 > ml-storage-ser22.nmg01: > getfattr: Removing leading '/' from absolute path names > # file: data3/bricks/project2/371_37829/test-dir > trusted.gfid=0x57e0a8945a224ab4be2c6c71eada6217 > trusted.glusterfs.dht=0xdbde0bb100000000f9e79e76fcf3cf38 > trusted.glusterfs.quota.dirty=0x3000 > trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1> 0x000000000000000000000000000000000000000000000001 > trusted.glusterfs.quota.size.1=0x0000000000000000000000000000 > 00000000000000000001 > ml-storage-ser29.nmg01: > getfattr: Removing leading '/' from absolute path names > # file: data3/bricks/project2/371_37829/test-dir > trusted.gfid=0x57e0a8945a224ab4be2c6c71eada6217 > trusted.glusterfs.dht=0xdbde0bb1000000001555555518618617 > trusted.glusterfs.quota.dirty=0x3000 > trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1> 0x000000000000000000000000000000000000000000000001 > trusted.glusterfs.quota.size.1=0x0000000000000000000000000000 > 00000000000000000001 > ml-storage-ser21.nmg01: > getfattr: Removing leading '/' from absolute path names > # file: data3/bricks/project2/371_37829/test-dir > trusted.gfid=0x57e0a8945a224ab4be2c6c71eada6217 > trusted.glusterfs.dht=0xdbde0bb100000000d8618615db6db6d7 > trusted.glusterfs.quota.dirty=0x3000 > trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1> 0x000000000000000000000000000000000000000000000001 > trusted.glusterfs.quota.size.1=0x0000000000000000000000000000 > 00000000000000000001 > ml-storage-ser27.nmg01: > getfattr: Removing leading '/' from absolute path names > # file: data3/bricks/project2/371_37829/test-dir > trusted.gfid=0x57e0a8945a224ab4be2c6c71eada6217 > trusted.glusterfs.dht=0xdbde0bb1000000001249249215555554 > trusted.glusterfs.quota.dirty=0x3000 > trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1> 0x000000000000000000000000000000000000000000000001 > trusted.glusterfs.quota.size.1=0x0000000000000000000000000000 > 00000000000000000001 > ml-storage-ser20.nmg01: > getfattr: Removing leading '/' from absolute path names > # file: data3/bricks/project2/371_37829/test-dir > trusted.gfid=0x57e0a8945a224ab4be2c6c71eada6217 > trusted.glusterfs.dht=0xdbde0bb100000000d8618615db6db6d7 > trusted.glusterfs.quota.dirty=0x3000 > trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1> 0x000000000000000000000000000000000000000000000001 > trusted.glusterfs.quota.size.1=0x0000000000000000000000000000 > 00000000000000000001 > > > > [root at ml-ctl-ser01 sunyuyusun]$ salt "ml-storage-ser2*.nmg01" cmd.run > "getfattr -d -m. -e hex /data4/bricks/project2/371_37829/test-dir" > ml-storage-ser25.nmg01: > getfattr: Removing leading '/' from absolute path names > # file: data4/bricks/project2/371_37829/test-dir > trusted.gfid=0x57e0a8945a224ab4be2c6c71eada6217 > trusted.glusterfs.dht=0xdbde0bb10000000024924924279e79e6 > trusted.glusterfs.quota.dirty=0x3000 > trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1> 0x000000000000000000000000000000000000000000000001 > trusted.glusterfs.quota.size.1=0x0000000000000000000000000000 > 00000000000000000001 > ml-storage-ser27.nmg01: > getfattr: Removing leading '/' from absolute path names > # file: data4/bricks/project2/371_37829/test-dir > trusted.gfid=0x57e0a8945a224ab4be2c6c71eada6217 > trusted.glusterfs.dht=0xdbde0bb100000000279e79e72aaaaaa9 > trusted.glusterfs.quota.dirty=0x3000 > trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1> 0x000000000000000000000000000000000000000000000001 > trusted.glusterfs.quota.size.1=0x0000000000000000000000000000 > 00000000000000000001 > ml-storage-ser28.nmg01: > getfattr: Removing leading '/' from absolute path names > # file: data4/bricks/project2/371_37829/test-dir > trusted.gfid=0x57e0a8945a224ab4be2c6c71eada6217 > trusted.glusterfs.dht=0xdbde0bb1000000002aaaaaaa2db6db6c > trusted.glusterfs.quota.dirty=0x3000 > trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1> 0x000000000000000000000000000000000000000000000001 > trusted.glusterfs.quota.size.1=0x0000000000000000000000000000 > 00000000000000000001 > ml-storage-ser23.nmg01: > getfattr: Removing leading '/' from absolute path names > # file: data4/bricks/project2/371_37829/test-dir > trusted.gfid=0x57e0a8945a224ab4be2c6c71eada6217 > trusted.glusterfs.dht=0xdbde0bb1000000002186186124924923 > trusted.glusterfs.quota.dirty=0x3000 > trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1> 0x000000000000000000000000000000000000000000000001 > trusted.glusterfs.quota.size.1=0x0000000000000000000000000000 > 00000000000000000001 > ml-storage-ser26.nmg01: > getfattr: Removing leading '/' from absolute path names > # file: data4/bricks/project2/371_37829/test-dir > trusted.gfid=0x57e0a8945a224ab4be2c6c71eada6217 > trusted.glusterfs.dht=0xdbde0bb100000000279e79e72aaaaaa9 > trusted.glusterfs.quota.dirty=0x3000 > trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1> 0x000000000000000000000000000000000000000000000001 > trusted.glusterfs.quota.size.1=0x0000000000000000000000000000 > 00000000000000000001 > ml-storage-ser29.nmg01: > getfattr: Removing leading '/' from absolute path names > # file: data4/bricks/project2/371_37829/test-dir > trusted.gfid=0x57e0a8945a224ab4be2c6c71eada6217 > trusted.glusterfs.dht=0xdbde0bb1000000002aaaaaaa2db6db6c > trusted.glusterfs.quota.dirty=0x3000 > trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1> 0x000000000000000000000000000000000000000000000001 > trusted.glusterfs.quota.size.1=0x0000000000000000000000000000 > 00000000000000000001 > ml-storage-ser22.nmg01: > getfattr: Removing leading '/' from absolute path names > # file: data4/bricks/project2/371_37829/test-dir > trusted.gfid=0x57e0a8945a224ab4be2c6c71eada6217 > trusted.glusterfs.dht=0xdbde0bb1000000002186186124924923 > trusted.glusterfs.quota.dirty=0x3000 > trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1> 0x000000000000000000000000000000000000000000000001 > trusted.glusterfs.quota.size.1=0x0000000000000000000000000000 > 00000000000000000001 > ml-storage-ser24.nmg01: > getfattr: Removing leading '/' from absolute path names > # file: data4/bricks/project2/371_37829/test-dir > trusted.gfid=0x57e0a8945a224ab4be2c6c71eada6217 > trusted.glusterfs.dht=0xdbde0bb10000000024924924279e79e6 > trusted.glusterfs.quota.dirty=0x3000 > trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1> 0x000000000000000000000000000000000000000000000001 > trusted.glusterfs.quota.size.1=0x0000000000000000000000000000 > 00000000000000000001 > ml-storage-ser20.nmg01: > getfattr: Removing leading '/' from absolute path names > # file: data4/bricks/project2/371_37829/test-dir > c=0x57e0a8945a224ab4be2c6c71eada6217 > trusted.glusterfs.dht=0xdbde0bb1000000001e79e79e21861860 > trusted.glusterfs.quota.dirty=0x3000 > trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1> 0x000000000000000000000000000000000000000000000001 > trusted.glusterfs.quota.size.1=0x0000000000000000000000000000 > 00000000000000000001 > ml-storage-ser21.nmg01: > getfattr: Removing leading '/' from absolute path names > # file: data4/bricks/project2/371_37829/test-dir > trusted.gfid=0x57e0a8945a224ab4be2c6c71eada6217 > trusted.glusterfs.dht=0xdbde0bb1000000001e79e79e21861860 > trusted.glusterfs.quota.dirty=0x3000 > trusted.glusterfs.quota.f8c0ee37-c980-4162-b7ed-15f911d0843a.contri.1> 0x000000000000000000000000000000000000000000000001 > trusted.glusterfs.quota.size.1=0x0000000000000000000000000000 > 00000000000000000001 > [root at ml-ctl-ser01 sunyuyusun]$ > > Is the infomation listed above what you need? I can see the two directory > (src and dst) are of diffenent trusted.gfid. >Yes. As you observed both directories have different gfids. So, this issue may not be related to to bugs I pointed out earlier.> also I have another test and basically certain that when i enable quota, > this will happen in one mount point(i only mount the test volume in one > mount point) > > > > Best Regards > Yours > Yu > > > > Raghavendra Gowdappa <rgowdapp at redhat.com> ?2018?9?5??? ??12:40??? > >> >> >> On Wed, Sep 5, 2018 at 9:21 AM, yu sun <sunyu1949 at gmail.com> wrote: >> >>> thank you , I will investigate and have a try. >>> >> >> Please get all the extended attributes of src and dst from all bricks. >> That should help us to (dis)prove whether its same bug. >> >> >>> BYW, I found another thing: >>> 1. I found this only occur when I try to mv dir to a subdir. >>> 2. I create a similar volume with default option and without quota >>> enabled, i found its ok, mv seems no problem. >>> >>> so I think it's possibly the volume option led to this problem, but I >>> dont know which options and how to resolve this, the volume have about 25T >>> data. >>> >>> Best Regards. >>> >>> Raghavendra Gowdappa <rgowdapp at redhat.com> ?2018?9?5??? ??10:50??? >>> >>>> >>>> >>>> On Tue, Sep 4, 2018 at 5:28 PM, yu sun <sunyu1949 at gmail.com> wrote: >>>> >>>>> Hi all: >>>>> >>>>> I have a replicated volume project2 with info: >>>>> Volume Name: project2 Type: Distributed-Replicate Volume ID: >>>>> 60175b8e-de0e-4409-81ae-7bb5eb5cacbf Status: Started Snapshot Count: >>>>> 0 Number of Bricks: 84 x 2 = 168 Transport-type: tcp Bricks: Brick1: >>>>> node20:/data2/bricks/project2 Brick2: node21:/data2/bricks/project2 Brick3: >>>>> node22:/data2/bricks/project2 Brick4: node23:/data2/bricks/project2 Brick5: >>>>> node24:/data2/bricks/project2 Brick6: node25:/data2/bricks/project2 Brick7: >>>>> node26:/data2/bricks/project2 Brick8: node27:/data2/bricks/project2 Brick9: >>>>> node28:/data2/bricks/project2 Brick10: node29:/data2/bricks/project2 >>>>> Brick11: node30:/data2/bricks/project2 Brick12: >>>>> node31:/data2/bricks/project2 Brick13: node32:/data2/bricks/project2 >>>>> Brick14: node33:/data2/bricks/project2 Brick15: >>>>> node20:/data3/bricks/project2 Brick16: node21:/data3/bricks/project2 >>>>> Brick17: node22:/data3/bricks/project2 Brick18: >>>>> node23:/data3/bricks/project2 Brick19: node24:/data3/bricks/project2 >>>>> Brick20: node25:/data3/bricks/project2 Brick21: >>>>> node26:/data3/bricks/project2 Brick22: node27:/data3/bricks/project2 >>>>> Brick23: node28:/data3/bricks/project2 Brick24: >>>>> node29:/data3/bricks/project2 Brick25: node30:/data3/bricks/project2 >>>>> Brick26: node31:/data3/bricks/project2 Brick27: >>>>> node32:/data3/bricks/project2 Brick28: node33:/data3/bricks/project2 >>>>> Brick29: node20:/data4/bricks/project2 Brick30: >>>>> node21:/data4/bricks/project2 Brick31: node22:/data4/bricks/project2 >>>>> Brick32: node23:/data4/bricks/project2 Brick33: >>>>> node24:/data4/bricks/project2 Brick34: node25:/data4/bricks/project2 >>>>> Brick35: node26:/data4/bricks/project2 Brick36: >>>>> node27:/data4/bricks/project2 Brick37: node28:/data4/bricks/project2 >>>>> Brick38: node29:/data4/bricks/project2 Brick39: >>>>> node30:/data4/bricks/project2 Brick40: node31:/data4/bricks/project2 >>>>> Brick41: node32:/data4/bricks/project2 Brick42: >>>>> node33:/data4/bricks/project2 Brick43: node20:/data5/bricks/project2 >>>>> Brick44: node21:/data5/bricks/project2 Brick45: >>>>> node22:/data5/bricks/project2 Brick46: node23:/data5/bricks/project2 >>>>> Brick47: node24:/data5/bricks/project2 Brick48: >>>>> node25:/data5/bricks/project2 Brick49: node26:/data5/bricks/project2 >>>>> Brick50: node27:/data5/bricks/project2 Brick51: >>>>> node28:/data5/bricks/project2 Brick52: node29:/data5/bricks/project2 >>>>> Brick53: node30:/data5/bricks/project2 Brick54: >>>>> node31:/data5/bricks/project2 Brick55: node32:/data5/bricks/project2 >>>>> Brick56: node33:/data5/bricks/project2 Brick57: >>>>> node20:/data6/bricks/project2 Brick58: node21:/data6/bricks/project2 >>>>> Brick59: node22:/data6/bricks/project2 Brick60: >>>>> node23:/data6/bricks/project2 Brick61: node24:/data6/bricks/project2 >>>>> Brick62: node25:/data6/bricks/project2 Brick63: >>>>> node26:/data6/bricks/project2 Brick64: node27:/data6/bricks/project2 >>>>> Brick65: node28:/data6/bricks/project2 Brick66: >>>>> node29:/data6/bricks/project2 Brick67: node30:/data6/bricks/project2 >>>>> Brick68: node31:/data6/bricks/project2 Brick69: >>>>> node32:/data6/bricks/project2 Brick70: node33:/data6/bricks/project2 >>>>> Brick71: node20:/data7/bricks/project2 Brick72: >>>>> node21:/data7/bricks/project2 Brick73: node22:/data7/bricks/project2 >>>>> Brick74: node23:/data7/bricks/project2 Brick75: >>>>> node24:/data7/bricks/project2 Brick76: node25:/data7/bricks/project2 >>>>> Brick77: node26:/data7/bricks/project2 Brick78: >>>>> node27:/data7/bricks/project2 Brick79: node28:/data7/bricks/project2 >>>>> Brick80: node29:/data7/bricks/project2 Brick81: >>>>> node30:/data7/bricks/project2 Brick82: node31:/data7/bricks/project2 >>>>> Brick83: node32:/data7/bricks/project2 Brick84: >>>>> node33:/data7/bricks/project2 Brick85: node20:/data8/bricks/project2 >>>>> Brick86: node21:/data8/bricks/project2 Brick87: >>>>> node22:/data8/bricks/project2 Brick88: node23:/data8/bricks/project2 >>>>> Brick89: node24:/data8/bricks/project2 Brick90: >>>>> node25:/data8/bricks/project2 Brick91: node26:/data8/bricks/project2 >>>>> Brick92: node27:/data8/bricks/project2 Brick93: >>>>> node28:/data8/bricks/project2 Brick94: node29:/data8/bricks/project2 >>>>> Brick95: node30:/data8/bricks/project2 Brick96: >>>>> node31:/data8/bricks/project2 Brick97: node32:/data8/bricks/project2 >>>>> Brick98: node33:/data8/bricks/project2 Brick99: >>>>> node20:/data9/bricks/project2 Brick100: node21:/data9/bricks/project2 >>>>> Brick101: node22:/data9/bricks/project2 Brick102: >>>>> node23:/data9/bricks/project2 Brick103: node24:/data9/bricks/project2 >>>>> Brick104: node25:/data9/bricks/project2 Brick105: >>>>> node26:/data9/bricks/project2 Brick106: node27:/data9/bricks/project2 >>>>> Brick107: node28:/data9/bricks/project2 Brick108: >>>>> node29:/data9/bricks/project2 Brick109: node30:/data9/bricks/project2 >>>>> Brick110: node31:/data9/bricks/project2 Brick111: >>>>> node32:/data9/bricks/project2 Brick112: node33:/data9/bricks/project2 >>>>> Brick113: node20:/data10/bricks/project2 Brick114: >>>>> node21:/data10/bricks/project2 Brick115: node22:/data10/bricks/project2 >>>>> Brick116: node23:/data10/bricks/project2 Brick117: >>>>> node24:/data10/bricks/project2 Brick118: node25:/data10/bricks/project2 >>>>> Brick119: node26:/data10/bricks/project2 Brick120: >>>>> node27:/data10/bricks/project2 Brick121: node28:/data10/bricks/project2 >>>>> Brick122: node29:/data10/bricks/project2 Brick123: >>>>> node30:/data10/bricks/project2 Brick124: node31:/data10/bricks/project2 >>>>> Brick125: node32:/data10/bricks/project2 Brick126: >>>>> node33:/data10/bricks/project2 Brick127: node20:/data11/bricks/project2 >>>>> Brick128: node21:/data11/bricks/project2 Brick129: >>>>> node22:/data11/bricks/project2 Brick130: node23:/data11/bricks/project2 >>>>> Brick131: node24:/data11/bricks/project2 Brick132: >>>>> node25:/data11/bricks/project2 Brick133: node26:/data11/bricks/project2 >>>>> Brick134: node27:/data11/bricks/project2 Brick135: >>>>> node28:/data11/bricks/project2 Brick136: node29:/data11/bricks/project2 >>>>> Brick137: node30:/data11/bricks/project2 Brick138: >>>>> node31:/data11/bricks/project2 Brick139: node32:/data11/bricks/project2 >>>>> Brick140: node33:/data11/bricks/project2 Brick141: >>>>> node20:/data12/bricks/project2 Brick142: node21:/data12/bricks/project2 >>>>> Brick143: node22:/data12/bricks/project2 Brick144: >>>>> node23:/data12/bricks/project2 Brick145: node24:/data12/bricks/project2 >>>>> Brick146: node25:/data12/bricks/project2 Brick147: >>>>> node26:/data12/bricks/project2 Brick148: node27:/data12/bricks/project2 >>>>> Brick149: node28:/data12/bricks/project2 Brick150: >>>>> node29:/data12/bricks/project2 Brick151: node30:/data12/bricks/project2 >>>>> Brick152: node31:/data12/bricks/project2 Brick153: >>>>> node32:/data12/bricks/project2 Brick154: node33:/data12/bricks/project2 >>>>> Brick155: node20:/data13/bricks/project2 Brick156: >>>>> node21:/data13/bricks/project2 Brick157: node22:/data13/bricks/project2 >>>>> Brick158: node23:/data13/bricks/project2 Brick159: >>>>> node24:/data13/bricks/project2 Brick160: node25:/data13/bricks/project2 >>>>> Brick161: node26:/data13/bricks/project2 Brick162: >>>>> node27:/data13/bricks/project2 Brick163: node28:/data13/bricks/project2 >>>>> Brick164: node29:/data13/bricks/project2 Brick165: >>>>> node30:/data13/bricks/project2 Brick166: node31:/data13/bricks/project2 >>>>> Brick167: node32:/data13/bricks/project2 Brick168: >>>>> node33:/data13/bricks/project2 Options Reconfigured: >>>>> performance.force-readdirp: on performance.write-behind: off >>>>> performance.stat-prefetch: on performance.client-io-threads: on >>>>> nfs.disable: on transport.address-family: inet features.quota: on >>>>> features.inode-quota: on features.quota-deem-statfs: on >>>>> cluster.readdir-optimize: on cluster.lookup-optimize: on >>>>> dht.force-readdirp: off client.event-threads: 10 server.event-threads: 10 >>>>> performance.readdir-ahead: on performance.io-cache: on >>>>> performance.flush-behind: on performance.cache-size: 5GB >>>>> performance.cache-max-file-size: 1MB performance.write-behind-window-size: >>>>> 10MB performance.read-ahead: off network.remote-dio: enable >>>>> performance.strict-o-direct: disable performance.io-thread-count: 25 >>>>> >>>>> >>>>> the volume looks ok, and I mount this volume on my client machine: >>>>> mount -t glusterfs -o oom-score-adj=-999 -o direct-io-mode=disable -o >>>>> use-readdirp=no node20:/project2 /mnt/project2 >>>>> >>>>> I have a directory in /mnt/project2/, but when I mv the directory to >>>>> other dirs, files in the dir lost while tree or ls the dir, some files >>>>> missing, my operations is list as below: >>>>> >>>> >>>> Looks very similar to: >>>> https://bugzilla.redhat.com/show_bug.cgi?id=1118762 >>>> https://bugzilla.redhat.com/show_bug.cgi?id=1337394 >>>> >>>> >>>> >>>>> root at ml-gpu-ser129.nmg01:/mnt/project2/371_37829$ mkdir test-dir >>>>> root at ml-gpu-ser129.nmg01:/mnt/project2/371_37829$ tree >>>>> . >>>>> ??? face_landmarks >>>>> ? ??? alive >>>>> ? ??? logs_20180823_28 >>>>> ? ??? info_000000.out >>>>> ? ??? info_000001.out >>>>> ? ??? info_000002.out >>>>> ? ??? info_000003.out >>>>> ? ??? info_000004.out >>>>> ? ??? info_000005.out >>>>> ? ??? info_000006.out >>>>> ? ??? info_000007.out >>>>> ? ??? info_000008.out >>>>> ? ??? info_000009.out >>>>> ? ??? info_000010.out >>>>> ? ??? info_000011.out >>>>> ? ??? info_000012.out >>>>> ? ??? info_000013.out >>>>> ? ??? info_000014.out >>>>> ? ??? info_000015.out >>>>> ? ??? info_000016.out >>>>> ? ??? info_000017.out >>>>> ? ??? info_000018.out >>>>> ? ??? info_000019.out >>>>> ??? test-dir >>>>> >>>>> 4 directories, 20 files >>>>> root at ml-gpu-ser129.nmg01:/mnt/project2/371_37829$ mv face_landmarks/ >>>>> test-dir/ >>>>> root at ml-gpu-ser129.nmg01:/mnt/project2/371_37829$ tree >>>>> . >>>>> ??? test-dir >>>>> ??? face_landmarks >>>>> >>>>> 2 directories, 0 files >>>>> root at ml-gpu-ser129.nmg01:/mnt/project2/371_37829$ cd >>>>> test-dir/face_landmarks/ >>>>> root at ml-gpu-ser129.nmg01:/mnt/project2/371_37829/test-dir/face_landmarks$ >>>>> ls >>>>> root at ml-gpu-ser129.nmg01:/mnt/project2/371_37829/test-dir/face_landmarks$ >>>>> cd .. >>>>> root at ml-gpu-ser129.nmg01:/mnt/project2/371_37829/test-dir$ mv >>>>> face_landmarks/ .. >>>>> root at ml-gpu-ser129.nmg01:/mnt/project2/371_37829/test-dir$ cd .. >>>>> root at ml-gpu-ser129.nmg01:/mnt/project2/371_37829$ tree >>>>> . >>>>> ??? face_landmarks >>>>> ? ??? alive >>>>> ??? test-dir >>>>> >>>>> 3 directories, 0 files >>>>> root at ml-gpu-ser129.nmg01:/mnt/project2/371_37829$ >>>>> >>>>> I think i make some mistakes with volume option, buti i am not sure, >>>>> so how can i find the lost files? the files seems still int the directory, >>>>> because i cant remove the directory and rm tell me "Not empty directory" >>>>> >>>> >>>> Its likely that src and dst of mv having same gfid and that's causing >>>> the issues. Can you look into both src and dst paths on all bricks? Union >>>> of contents of both directories should give all the files in the src >>>> directory before mv. Once found you can, >>>> * keep a backup of contents of src and dst on all bricks >>>> * remove trusted.gfid xattr on src and dst from all bricks >>>> * remove gfid handle (.glusterfs/<first two characters of gfid>/<second >>>> set of two characters of gfid>/<gfid> on each brick) >>>> * disable readdirplus in entire stack (maybe you can use a tmp mount >>>> for this) [1] >>>> * stat src and dst on a mount point with readdirplus disabled. >>>> * Now you'll see two directories src and dst on mountpoint. You can >>>> copy the contents of both into a new directory >>>> >>>> [1] https://lists.gluster.org/pipermail/gluster-users/2017- >>>> March/030148.html >>>> >>>> >>>>> >>>>> Any suggestions is appreciated. >>>>> Many Thanks >>>>> >>>>> Best regards >>>>> Yu >>>>> >>>>> _______________________________________________ >>>>> Gluster-users mailing list >>>>> Gluster-users at gluster.org >>>>> https://lists.gluster.org/mailman/listinfo/gluster-users >>>>> >>>> >>>> >>-------------- next part -------------- An HTML attachment was scrubbed... URL: <http://lists.gluster.org/pipermail/gluster-users/attachments/20180905/f2062c84/attachment-0001.html>