thr3ads.net - Gluster users - [Gluster-users] Possible memory leak in gluster samba vfs [Sep 2013]

If this information is useful, please help other people find it:
Share via:

kane

2013-Sep-13 12:33 UTC

[Gluster-users] Possible memory leak in gluster samba vfs

Hi

	We use samba gluster vfs in IO test, but meet with gluster server smbd oom
killer,
The smbd process spend over 15g RES with top command show, in the end is our
simple test code:

gluster server vfs --> smbd --> client mount dir
"/mnt/vfs"--> execute vfs test program "$ ./vfs 1000"

then we can watch gluster server smbd RES with top command.

  PID USER      PR  NI  VIRT  RES  SHR S %CPU %MEM    TIME+  COMMAND
 4000 soul      20   0 5486m 4.9g  10m R 108.4 31.5 111:07.07 smbd
 3447 root      20   0 1408m  44m 2428 S 44.4  0.3  59:11.55 glusterfsd   

io test code:
======================================#define _LARGEFILE64_SOURCE

#include <stdio.h>
#include <unistd.h>
#include <string.h>
#include <pthread.h>
#include <stdlib.h>
#include <fcntl.h>
#include <sys/types.h>

int WT = 1;

#define RND(x) ((x>0)?(genrand() % (x)):0)
extern unsigned long genrand();
extern void sgenrand();

/* Period parameters */  
#define N 624
#define M 397
#define MATRIX_A 0x9908b0df   /* constant vector a */
#define UPPER_MASK 0x80000000 /* most significant w-r bits */
#define LOWER_MASK 0x7fffffff /* least significant r bits */

/* Tempering parameters */   
#define TEMPERING_MASK_B 0x9d2c5680
#define TEMPERING_MASK_C 0xefc60000
#define TEMPERING_SHIFT_U(y)  (y >> 11)
#define TEMPERING_SHIFT_S(y)  (y << 7)
#define TEMPERING_SHIFT_T(y)  (y << 15)
#define TEMPERING_SHIFT_L(y)  (y >> 18)

static unsigned long mt[N]; /* the array for the state vector  */
static int mti=N+1; /* mti==N+1 means mt[N] is not initialized */

/* Initializing the array with a seed */
void
sgenrand(seed)
    unsigned long seed;	
{
    int i;

    for (i=0;i<N;i++) {
         mt[i] = seed & 0xffff0000;
         seed = 69069 * seed + 1;
         mt[i] |= (seed & 0xffff0000) >> 16;
         seed = 69069 * seed + 1;
    }
    mti = N;
}

unsigned long 
genrand()
{
    unsigned long y;
    static unsigned long mag01[2]={0x0, MATRIX_A};
    /* mag01[x] = x * MATRIX_A  for x=0,1 */

    if (mti >= N) { /* generate N words at one time */
        int kk;

        if (mti == N+1)   /* if sgenrand() has not been called, */
            sgenrand(4357); /* a default initial seed is used   */

        for (kk=0;kk<N-M;kk++) {
            y = (mt[kk]&UPPER_MASK)|(mt[kk+1]&LOWER_MASK);
            mt[kk] = mt[kk+M] ^ (y >> 1) ^ mag01[y & 0x1];
        }
        for (;kk<N-1;kk++) {
            y = (mt[kk]&UPPER_MASK)|(mt[kk+1]&LOWER_MASK);
            mt[kk] = mt[kk+(M-N)] ^ (y >> 1) ^ mag01[y & 0x1];
        }
        y = (mt[N-1]&UPPER_MASK)|(mt[0]&LOWER_MASK);
        mt[N-1] = mt[M-1] ^ (y >> 1) ^ mag01[y & 0x1];

        mti = 0;
    }
  
    y = mt[mti++];
    y ^= TEMPERING_SHIFT_U(y);
    y ^= TEMPERING_SHIFT_S(y) & TEMPERING_MASK_B;
    y ^= TEMPERING_SHIFT_T(y) & TEMPERING_MASK_C;
    y ^= TEMPERING_SHIFT_L(y);

    return y; 
}

char *initialize_file_source(int size)
{
   char *new_source;
   int i;

   if ((new_source=(char *)malloc(size))==NULL) /* allocate buffer */
      fprintf(stderr,"Error: failed to allocate source file of size
%d\n",size);
   else
      for (i=0; i<size; i++) /* file buffer with junk */
         new_source[i]=32+RND(95);

   return(new_source);
}

void *tran_file(void *map)
{
	int block_size = 512;
	char *read_buffer; /* temporary space for reading file data into */
	int fd = open((char *)map, O_RDWR | O_CREAT | O_TRUNC, 0644);
	if(fd == -1) {
		perror("open");
		return ;
	}

//	read_buffer=(char *)malloc(block_size);
//	memset(read_buffer, 0x56, block_size);
	read_buffer = initialize_file_source(block_size);
	while(WT){
		sleep(1);
		pwrite(fd, read_buffer, block_size, 0);
		pread(fd, read_buffer, block_size, 0);
	}
	free(read_buffer);
	close(fd);
}

int main(int argc, char *argv[])
{
	if(argc == 1) {
		printf("Set file name and file size in GB!\n");
		return -1;
	}

	unsigned long long length = atoll(argv[1]) * 1024 * 1024 * 1024;
	unsigned long long index = 0;
	unsigned long long count = 0;
	int fd = 0;
	char path[32];
	char *swap = NULL;
	int psize = 65536;
	int ret = 0;
	int ret_i = 0;
	void *tret;
	pthread_t rtid_1;
	pthread_t rtid_2;
	pthread_t rtid_3;
	pthread_t rtid_4;
	int fd_i = 0;
	int a_size = 256;
	char *append_buffer; /* temporary space for reading file data into */

	swap = (char *)malloc(sizeof(char) * psize);
	if(swap == NULL) {
		printf("Malloc Swap Failed!\n");
		close(fd);
		return -1;
	}
	memset(swap, 0x55, psize);

	append_buffer = initialize_file_source(a_size);

	pthread_create(&rtid_1, NULL, tran_file, "/mnt/vfs/a.map");
	pthread_create(&rtid_2, NULL, tran_file, "/mnt/vfs/b.map");
	pthread_create(&rtid_3, NULL, tran_file, "/mnt/vfs/c.map");
	pthread_create(&rtid_4, NULL, tran_file, "/mnt/vfs/d.map");

	while(index < length)
	{
		if((index % (1024*1024*1024)) == 0) {
			count++;
			sprintf(path, "/mnt/vfs/data/%08lld.dat", count);
			if((fd = open(path, O_RDWR | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR)) < 0)
			{
				printf("Can not open io file: %s\n", path);
				perror("open");
				return -1;
			}
			sprintf(path, "/mnt/vfs/data/%08lld.index", count);
			if((fd_i = open(path, O_RDWR | O_CREAT | O_APPEND, S_IRUSR | S_IWUSR)) <
0)
			{
				printf("Can not open io file: %s\n", path);
				perror("open");
				return -1;
			}
			printf("...1GiB\n");
		}

		ret = pwrite(fd, swap, psize, (index % (1024*1024*1024)));
		if(ret != psize) {
			perror("write");
			printf("Write data file failed!\n");
			continue;
		}
		lseek64(fd_i, ((index*a_size)/65536), SEEK_SET);
		ret_i = write(fd_i, append_buffer, a_size);
		if(ret_i != a_size) {
			perror("write");
			printf("Write index file failed!\n");
			continue;
		}

		index += ret;
		if((index % (1024*1024*1024)) == 0) {
			close(fd);
			close(fd_i);
		}
	}

	WT = 0;
	pthread_join(rtid_1, &tret);
	pthread_join(rtid_2, &tret);
	pthread_join(rtid_3, &tret);
	pthread_join(rtid_4, &tret);
	printf("%s GB write\n", argv[1]);
	sync();
	free(swap);
	free(append_buffer);

	return 0;
}

======================================



kane               
----------------------------------------------------------------
Email:  kai.zhou at soulinfo.com
??:    0510-85385788-616


-------------- next part --------------
An HTML attachment was scrubbed...
URL:
<http://supercolony.gluster.org/pipermail/gluster-users/attachments/20130913/7afbe394/attachment.html>

Lalatendu Mohanty

2013-Sep-13 13:52 UTC

head link

[Gluster-users] Possible memory leak in gluster samba vfs

On 09/13/2013 06:03 PM, kane wrote:> Hi
>
> We use samba gluster vfs in IO test, but meet with gluster server smbd
> oom killer,
> The smbd process spend over 15g RES with top command show, in the end
> is our simple test code:
>
> gluster server vfs --> smbd --> client mount dir
"/mnt/vfs"--> execute
> vfs test program "$ ./vfs 1000"
>
> then we can watch gluster server smbd RES with top command.
>
> PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND
> 4000 soul 20 0 5486m 4.9g 10m R 108.4 31.5 111:07.07 smbd
> 3447 root 20 0 1408m 44m 2428 S 44.4 0.3 59:11.55 glusterfsd
>
> io test code:
> ======================================> #define _LARGEFILE64_SOURCE
>
> #include <stdio.h>
> #include <unistd.h>
> #include <string.h>
> #include <pthread.h>
> #include <stdlib.h>
> #include <fcntl.h>
> #include <sys/types.h>
>
> int WT = 1;
>
> #define RND(x) ((x>0)?(genrand() % (x)):0)
> extern unsigned long genrand();
> extern void sgenrand();
>
> /* Period parameters */
> #define N 624
> #define M 397
> #define MATRIX_A 0x9908b0df /* constant vector a */
> #define UPPER_MASK 0x80000000 /* most significant w-r bits */
> #define LOWER_MASK 0x7fffffff /* least significant r bits */
>
> /* Tempering parameters */
> #define TEMPERING_MASK_B 0x9d2c5680
> #define TEMPERING_MASK_C 0xefc60000
> #define TEMPERING_SHIFT_U(y) (y >> 11)
> #define TEMPERING_SHIFT_S(y) (y << 7)
> #define TEMPERING_SHIFT_T(y) (y << 15)
> #define TEMPERING_SHIFT_L(y) (y >> 18)
>
> static unsigned long mt[N]; /* the array for the state vector */
> static int mti=N+1; /* mti==N+1 means mt[N] is not initialized */
>
> /* Initializing the array with a seed */
> void
> sgenrand(seed)
> unsigned long seed;
> {
> int i;
>
> for (i=0;i<N;i++) {
> mt[i] = seed & 0xffff0000;
> seed = 69069 * seed + 1;
> mt[i] |= (seed & 0xffff0000) >> 16;
> seed = 69069 * seed + 1;
> }
> mti = N;
> }
>
> unsigned long
> genrand()
> {
> unsigned long y;
> static unsigned long mag01[2]={0x0, MATRIX_A};
> /* mag01[x] = x * MATRIX_A for x=0,1 */
>
> if (mti >= N) { /* generate N words at one time */
> int kk;
>
> if (mti == N+1) /* if sgenrand() has not been called, */
> sgenrand(4357); /* a default initial seed is used */
>
> for (kk=0;kk<N-M;kk++) {
> y = (mt[kk]&UPPER_MASK)|(mt[kk+1]&LOWER_MASK);
> mt[kk] = mt[kk+M] ^ (y >> 1) ^ mag01[y & 0x1];
> }
> for (;kk<N-1;kk++) {
> y = (mt[kk]&UPPER_MASK)|(mt[kk+1]&LOWER_MASK);
> mt[kk] = mt[kk+(M-N)] ^ (y >> 1) ^ mag01[y & 0x1];
> }
> y = (mt[N-1]&UPPER_MASK)|(mt[0]&LOWER_MASK);
> mt[N-1] = mt[M-1] ^ (y >> 1) ^ mag01[y & 0x1];
>
> mti = 0;
> }
> y = mt[mti++];
> y ^= TEMPERING_SHIFT_U(y);
> y ^= TEMPERING_SHIFT_S(y) & TEMPERING_MASK_B;
> y ^= TEMPERING_SHIFT_T(y) & TEMPERING_MASK_C;
> y ^= TEMPERING_SHIFT_L(y);
>
> return y;
> }
>
> char *initialize_file_source(int size)
> {
> char *new_source;
> int i;
>
> if ((new_source=(char *)malloc(size))==NULL) /* allocate buffer */
> fprintf(stderr,"Error: failed to allocate source file of size
%d\n",size);
> else
> for (i=0; i<size; i++) /* file buffer with junk */
> new_source[i]=32+RND(95);
>
> return(new_source);
> }
>
> void *tran_file(void *map)
> {
> int block_size = 512;
> char *read_buffer; /* temporary space for reading file data into */
> int fd = open((char *)map, O_RDWR | O_CREAT | O_TRUNC, 0644);
> if(fd == -1) {
> perror("open");
> return ;
> }
>
> //read_buffer=(char *)malloc(block_size);
> //memset(read_buffer, 0x56, block_size);
> read_buffer = initialize_file_source(block_size);
> while(WT){
> sleep(1);
> pwrite(fd, read_buffer, block_size, 0);
> pread(fd, read_buffer, block_size, 0);
> }
> free(read_buffer);
> close(fd);
> }
>
> int main(int argc, char *argv[])
> {
> if(argc == 1) {
> printf("Set file name and file size in GB!\n");
> return -1;
> }
>
> unsigned long long length = atoll(argv[1]) * 1024 * 1024 * 1024;
> unsigned long long index = 0;
> unsigned long long count = 0;
> int fd = 0;
> char path[32];
> char *swap = NULL;
> int psize = 65536;
> int ret = 0;
> int ret_i = 0;
> void *tret;
> pthread_t rtid_1;
> pthread_t rtid_2;
> pthread_t rtid_3;
> pthread_t rtid_4;
> int fd_i = 0;
> int a_size = 256;
> char *append_buffer; /* temporary space for reading file data into */
>
> swap = (char *)malloc(sizeof(char) * psize);
> if(swap == NULL) {
> printf("Malloc Swap Failed!\n");
> close(fd);
> return -1;
> }
> memset(swap, 0x55, psize);
>
> append_buffer = initialize_file_source(a_size);
>
> pthread_create(&rtid_1, NULL, tran_file, "/mnt/vfs/a.map");
> pthread_create(&rtid_2, NULL, tran_file, "/mnt/vfs/b.map");
> pthread_create(&rtid_3, NULL, tran_file, "/mnt/vfs/c.map");
> pthread_create(&rtid_4, NULL, tran_file, "/mnt/vfs/d.map");
>
> while(index < length)
> {
> if((index % (1024*1024*1024)) == 0) {
> count++;
> sprintf(path, "/mnt/vfs/data/%08lld.dat", count);
> if((fd = open(path, O_RDWR | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR)) < 0)
> {
> printf("Can not open io file: %s\n", path);
> perror("open");
> return -1;
> }
> sprintf(path, "/mnt/vfs/data/%08lld.index", count);
> if((fd_i = open(path, O_RDWR | O_CREAT | O_APPEND, S_IRUSR | S_IWUSR))
> < 0)
> {
> printf("Can not open io file: %s\n", path);
> perror("open");
> return -1;
> }
> printf("...1GiB\n");
> }
>
> ret = pwrite(fd, swap, psize, (index % (1024*1024*1024)));
> if(ret != psize) {
> perror("write");
> printf("Write data file failed!\n");
> continue;
> }
> lseek64(fd_i, ((index*a_size)/65536), SEEK_SET);
> ret_i = write(fd_i, append_buffer, a_size);
> if(ret_i != a_size) {
> perror("write");
> printf("Write index file failed!\n");
> continue;
> }
>
> index += ret;
> if((index % (1024*1024*1024)) == 0) {
> close(fd);
> close(fd_i);
> }
> }
>
> WT = 0;
> pthread_join(rtid_1, &tret);
> pthread_join(rtid_2, &tret);
> pthread_join(rtid_3, &tret);
> pthread_join(rtid_4, &tret);
> printf("%s GB write\n", argv[1]);
> sync();
> free(swap);
> free(append_buffer);
>
> return 0;
> }
>
> ======================================>
>
>
>
> kane
> ----------------------------------------------------------------
> Email: kai.zhou at soulinfo.com <mailto:kai.zhou at soulinfo.com>
> ??: 0510-85385788-616
>
>Thanks for the code. I will try to reproduce the issue using your code.

-Lala
-------------- next part --------------
An HTML attachment was scrubbed...
URL:
<http://supercolony.gluster.org/pipermail/gluster-users/attachments/20130913/1dba3118/attachment.html>

Gluster users - Sep 2013 - Possible memory leak in gluster samba vfs

[Gluster-users] Possible memory leak in gluster samba vfs

[Gluster-users] Possible memory leak in gluster samba vfs