I'm having an issue where glusterfs is reporting the incorrect size for
recently changed files.
On client A, a process acquires a write lock on a file (using fcntl) and
begins to modify the file. On client B, a process attempts to a aquire a
write lock on the same file and blocks. The process on client A
finishes its modifications, calls fsync, and releases the lock. The
process on client B then successfully acquires the lock and reads the
file. At this point while the contents of the file match what was
written on client A, the size of the file is still reported as the old size.
The program below can be used to demonstrate the problem. It repeatedly
locks a file, reads in the whole file, checks that a string at the
beginning of the file that indicates the expected file size matches the
actual size read, and then truncates the file and writes out new data of
a random size. Running it on the same file from two different clients I
get things like...
Client A:
locked
Old size: 531
New size: 459
unlocked
Client B:
locked
Old size: 799
New size: 531
unlocked
locked
Short read. Expected 531 but got 459
File size now reported as 531
Size indicated in file is 459
Press enter to exit
After client A creates a 459 byte file and releases the lock, when
client B accesses the file, seeking to the end of the file reports the
end as 531 (the previous size created by client B), while a read of the
file only returns 459 bytes, and the string at the beginning of the file
is the 459 written by client A.
If I add a sleep(1) after acquiring the lock then everything works as
expected.
I'm using gluster 3.4.1, with a single server running CentOS 6.5, with
multiple bricks (no replication) formatted with xfs. The clients are
running Fedora 19. Filesystems mounted with
attribute-timeout=0,entry-timeout=0,acl,selinux. Is there some other
option I need to maintain client file metadata coherence?
#include <unistd.h>
#include <fcntl.h>
#include <iostream>
#include <stdlib.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <string>
#include <string.h>
#include <errno.h>
#include <sstream>
#include <vector>
void errorExit()
{
std::cout << "Press enter to exit" << std::endl;
std::cin.ignore();
exit(EXIT_FAILURE);
}
void makeFile(int fd, int fileSize)
{
if (ftruncate(fd, 0)==-1)
{
std::cerr << "Can't truncate file. " <<
strerror(errno) <<
std::endl;
errorExit();
}
std::ostringstream buf;
buf << fileSize << std::endl;
while(buf.str().size() < fileSize)
buf << "0";
if (lseek(fd, 0, SEEK_SET) == -1)
{
std::cerr << "Can't seek in file. " <<
strerror(errno) <<
std::endl;
errorExit();
}
if (write(fd, buf.str().c_str(), buf.str().size()) == -1)
{
std::cerr << "Can't write to file. " <<
strerror(errno) <<
std::endl;
errorExit();
}
if (fsync(fd) == -1)
{
std::cerr << "Can't write to file. " <<
strerror(errno) <<
std::endl;
errorExit();
}
}
int main(int argc, const char* argv[])
{
std::string filePath = std::string(argv[1]);
struct stat statBuf;
if (stat(filePath.c_str(), &statBuf) != 0)
{
int fd = open(filePath.c_str(), O_RDWR | O_CREAT, S_IRUSR |
S_IWUSR);
if (fd == -1)
{
std::cerr << "Couldn't create file" <<
strerror(errno) <<
std::endl;
return EXIT_FAILURE;
}
int s = (rand() % 1024) + 100;
makeFile(fd, s);
close(fd);
}
while(true)
{
int fd = open(filePath.c_str(), O_RDWR);
if (fd == -1)
{
std::cerr << "Couldn't open file" <<
std::endl;
errorExit();
}
struct flock lock;
lock.l_start = 0;
lock.l_whence = SEEK_SET;
lock.l_len = 0;
lock.l_type = F_WRLCK;
if (fcntl(fd, F_SETLKW, &lock)==-1)
{
std::cerr << "couldn't get lock" <<
std::endl;
errorExit();
}
std::cout << "locked" << std::endl;
off_t fileSize = lseek(fd, 0, SEEK_END);
if (fileSize == -1)
{
std::cerr << "Couldn't get file size" <<
strerror(errno) <<
std::endl;
errorExit();
}
if (lseek(fd, 0, SEEK_SET) == -1)
{
std::cerr << "Couldn't seek to beginning of
file." <<
strerror(errno) << std::endl;
errorExit();
}
std::vector<char> buffer(fileSize);
ssize_t bytesRead = read(fd, &buffer[0], fileSize);
if (bytesRead == -1)
{
std::cerr << "Error reading file." <<
strerror(errno) <<
std::endl;
errorExit();
}
std::istringstream instream(std::string(&buffer[0],
buffer.size()));
if (bytesRead != fileSize)
{
std::cerr << "Short read. Expected " <<
fileSize << " but
got " << bytesRead << std::endl;
fileSize = lseek(fd, 0, SEEK_END);
std::cerr << "File size now reported as " <<
fileSize <<
std::endl;
int s;
instream >> s;
std::cerr << "Size indicated in file is " <<
s << std::endl;
errorExit();
}
int expectedSize;
instream >> expectedSize;
if (expectedSize != fileSize)
{
std::cerr << "Expected " << expectedSize
<< " bytes in file
but found " << fileSize << std::endl;
errorExit();
}
std::cout << "Old size: " << fileSize <<
std::endl;
int newSize = (rand() % 1024) + 100;
std::cout << "New size: " << newSize <<
std::endl;
makeFile(fd, newSize);
lock.l_type = F_UNLCK;
if (fcntl(fd, F_SETLKW, &lock)==-1)
{
std::cout << "couldn't release lock" <<
std::endl;
errorExit();
}
std::cout << "unlocked" << std::endl;
close(fd);
}
}
--
Jason Ferrara
Jacquette Consulting, Inc.
710 Providence Road
Malvern, PA 19355
jason.ferrara at jacquette.com