search for: map_struct

...vior, and acting accordingly in a few locations. 2. The code and usage of map_ptr() required the file size to be known in advance. This is impossible for devices and FIFOs. To address this, I changed the interface of map_ptr. It now returns void rather than char*, and updates two new members of map_struct: - m_ptr contains the pointer to the new data - m_len contains the length of the new data (at most the length requested, but less if EOF encountered) Also, the map_struct->file_size field is now updated by map_ptr() when it encounters an EOF. I updated all invocations of map_ptr() to use the...

Attached is a patch that makes window strides constant when files are walked with a constant block size. In these cases, it completely avoids all memmoves. In my simple local test of rsyncing 57MB of 10 local files, memmoved bytes went from 18MB to zero. I haven't tested this for a big variety of file cases. I think that this will always reduce the memmoves involved with walking a large

...f_out, block_size); - write_int(f_out, 0); - } + /* we don't have checksums */ + write_int(f_out, 0); + write_int(f_out, block_size); + write_int(f_out, 0); } @@ -162,57 +146,43 @@ generate approximately one checksum every n bytes */ -static struct sum_struct *generate_sums(struct map_struct *buf,OFF_T len,int n) +void generate_and_send_sums(struct map_struct *buf,OFF_T len,int block_len, int f_out) { - int i; - struct sum_struct *s; - int count; - int block_len = n; - int remainder = (len%block_len); + size_t i; + struct sum_struct sum; OFF_T offset = 0; - count = (len+(block_len...

Hi, I was trying to synchronize data sitting on our Sun Solaris 8 server with data size of about 800 GB to a remote Linux server in our LAN. Either the rsync process hangs for ever else I get "core dump" after about 15 minutes of time on the source host (solaris 8 server). I used rsync 2.6.2 using the basic command options as shown below # rsync -avz -e rsh <source folder

Maybe security related mails should be sent elsewhere? I didn't notice any so here it goes: sender.c:receive_sums() s->count = read_int(f); .. s->sums = (struct sum_buf *)malloc(sizeof(s->sums[0])*s->count); if (!s->sums) out_of_memory("receive_sums"); for (i=0; i < (int) s->count;i++) { s->sums[i].sum1 = read_int(f);

...ern int def_compress_level; @@ -624,6 +626,244 @@ static void see_deflate_token(char *buf, int32 len) #endif } +#ifdef HAVE_LIBZSTD + +static ZSTD_inBuffer zstd_in_buff; +static ZSTD_outBuffer zstd_out_buff; +static ZSTD_CCtx *zstd_cctx; + +static void send_zstd_token(int f, int32 token, struct map_struct *buf, + OFF_T offset, int32 nb) +{ + static int comp_init_done, flush_pending; + ZSTD_EndDirective flush = ZSTD_e_continue; + int32 n, r; + + /* initialization */ + if (!comp_init_done) { + + zstd_cctx = ZSTD_createCCtx(); + if (!zstd_cctx) { + rprintf(FERROR, "compression init faile...

...our bug report made the problem easy to find. I've checked in a fix to CVS, but you can fix your current source by changing the "size_t" to "OFF_T" in the generate_and_send_sums() function in generator.c. It should look like this: static void generate_and_send_sums(struct map_struct *buf, OFF_T len, int f_out) Then, recompile, install, and the problem should be gone. -- Configure bugmail: https://bugzilla.samba.org/userprefs.cgi?tab=email ------- You are receiving this mail because: ------- You are the QA contact for the bug, or are watching the QA contact.

...char *sum) +void get_checksum2(const void *buf,int len,unsigned char *sum) { int i; static char *buf1; @@ -82,7 +83,7 @@ } -void file_checksum(char *fname,char *sum,OFF_T size) +void file_checksum(const char *fname, void *sum,OFF_T size) { OFF_T i; struct map_struct *buf; @@ -133,15 +134,17 @@ void sum_init(void) { - char s[4]; + unsigned char s[4]; mdfour_begin(&md); sumresidue=0; SIVAL(s,0,checksum_seed); sum_update(s,4); } -void sum_update(char *p,int len) +void sum_update(void *p1,int len) { +un...

...@@ -165,6 +166,7 @@ static int receive_data(int f_in, char *fname_r, int fd_r, OFF_T size_r, const char *fname, int fd, OFF_T total_size) { + STRUCT_STAT st; static char file_sum1[MAX_DIGEST_LEN]; static char file_sum2[MAX_DIGEST_LEN]; struct map_struct *mapbuf; @@ -285,10 +287,14 @@ goto report_write_error; #ifdef HAVE_FTRUNCATE - if (inplace && fd != -1 - && ftruncate(fd, offset) < 0) { - rsyserr(FERROR_XFER, errno, "ftruncate failed on %s", - full_fn...

...Component: core AssignedTo: wayned at samba.org ReportedBy: an.m at outlook.com QAContact: rsync-qa at samba.org map_ptr() handles read failures by zeroing out remaining chunk and returning the partially filled/all zero buffers to callers. It also fills error details in map_struct->status. It is caller's responsibility to check the status and handle this condition appropriately. It is probably OK to ignore this when we are generating checksums (as it might lead to a false mismatch, resulting in transfer of a non matching block) But in context of transfer (send_defl...

...ackups; +extern int direct_write; extern char *backup_suffix; static struct delete_list { @@ -302,7 +303,8 @@ int fd1,fd2; STRUCT_STAT st; char *fname; - char fnametmp[MAXPATHLEN]; + char *fnametmp; + char fnametmpbuf[MAXPATHLEN]; char *fnamecmp; char fnamecmpbuf[MAXPATHLEN]; struct map_struct *buf; @@ -314,6 +316,7 @@ extern int preserve_perms; extern int delete_after; struct stats initial_stats; + int write_flags; if (verbose > 2) { rprintf(FINFO,"recv_files(%d) starting\n",flist->count); @@ -404,22 +407,29 @@ buf = NULL; } - if (!get_tmpname(fnam...

...long) sum->blength, (long) sum->s2length, + (double) sum->flength); + } +} /** * Perhaps we want to just send an empty checksum set for this file, @@ -163,30 +227,18 @@ * * Generate approximately one checksum every block_len bytes. */ -static void generate_and_send_sums(struct map_struct *buf, OFF_T len, - int block_len, int f_out) +static void generate_and_send_sums(struct map_struct *buf, OFF_T len, int f_out) { size_t i; struct sum_struct sum; OFF_T offset = 0; - sum.count = (len + (block_len - 1)) / block_len; - sum.remainder = (len % block_len); - sum.blength =...

> Just curious: what about the rsync+ patch? Thanks for the reminder. I've just committed Jos's rsync+ patch onto the "branch_mbp_rsyncplus_merge" branch. If it works OK and nobody screams I will move it across onto the main tree tomorrow or Wednesday. I see the patch doesn't add documentation about the new options to the man page, so we should fix that in the future.

...or working socketpair],rsync_cv_HAVE_SOCKETPAIR,[ AC_TRY_RUN([ diff -urN rsync-2.5.6.org/receiver.c rsync-2.5.6/receiver.c --- rsync-2.5.6.org/receiver.c Tue Jan 21 00:32:17 2003 +++ rsync-2.5.6/receiver.c Thu Feb 20 20:28:14 2003 @@ -203,7 +203,7 @@ } -static int receive_data(int f_in,struct map_struct *buf,int fd,char *fname, +static int recv_data(int f_in,struct map_struct *buf,int fd,char *fname, OFF_T total_size) { int i; @@ -383,14 +383,14 @@ if (fd1 != -1 && do_fstat(fd1,&st) != 0) { rprintf(FERROR,"fstat %s : %s\n",fnamecmp,strerror(errno)); - receiv...

People of cygwin & rsync, I recently attempted to get cygwin and rsync working to solve a backup/mirroring need in my computer life. Well, as you might guess, I ran into a little but of trouble. Strangely enough, rsync seemed to be regularly hanging when I attempted to do a "get" (sycronize a remote to a local dir). Well, considering I want to automate this, that was not going

...@ -35,7 +35,7 @@ extern int cvs_exclude; extern int io_error; extern char *tmpdir; -extern char *compare_dest; +extern char *compare_dest[]; extern int make_backups; extern int do_progress; extern char *backup_dir; @@ -295,7 +295,7 @@ char *fnamecmp; char fnamecmpbuf[MAXPATHLEN]; struct map_struct *mapbuf; - int i; + int i, j; struct file_struct *file; int phase=0; int recv_ok; @@ -361,12 +361,14 @@ /* open the file */ fd1 = do_open(fnamecmp, O_RDONLY, 0); - if (fd1 == -1 && compare_dest != NULL) { + j = 0; + while (fd1 == -1 && compare_dest[j] != NULL) {...

...16 */ - if (ret < bsize) ret = bsize; - if (ret > CHUNK_SIZE/2) ret = CHUNK_SIZE/2; - return ret; -} - /* * NULL sum_struct means we have no checksums @@ -156,24 +141,50 @@ * * Generate approximately one checksum every block_len bytes. */ -static void generate_and_send_sums(struct map_struct *buf, OFF_T len, - int block_len, int f_out) +static void generate_and_send_sums(struct map_struct *buf, OFF_T len, int f_out) { + extern int block_size; + int block_len; size_t i; struct sum_struct sum; OFF_T offset = 0; + /* adapt block size if not fixed */ + if (block_size) + { +...

...if (whole_file == 0) return False; return local_server; } *************** *** 227,242 **** * Generate and send a stream of signatures/checksums that describe a buffer * * Generate approximately one checksum every block_len bytes. */ ! static void generate_and_send_sums(struct map_struct *buf, size_t len, int f_out) { ! size_t i; struct sum_struct sum; OFF_T offset = 0; sum_sizes_sqroot(&sum, len); write_sum_head(f_out, &sum); for (i = 0; i < sum.count; i++) { unsigned int n1 = MIN(len, sum.blength); --- 228,253 ---- * Generate and send...

https://bugzilla.samba.org/show_bug.cgi?id=7876 Summary: please implement o_direct Product: rsync Version: 3.1.0 Platform: Other OS/Version: Linux Status: NEW Severity: normal Priority: P3 Component: core AssignedTo: wayned at samba.org ReportedBy: costinel at gmail.com QAContact:

In short: Platform: linux with 2.4 kernel Version: rsync 2.6.6 Command line: rsync266 -av -W --bwlimit=1 /mnt/somedir/rsync-2.6.6.tar.gz ./ Destination: local disk Source: file on a smbfs mounted filesystem; share is exported on a NT 4.0 workstation over a very slow and unstable link Result: Rsync completes operation with no special message, but the resulting file is damaged, large