Hi,
I have written a 'smallish' patch to implement the --inplace option
as discussed on this mailing list at various points in the past. It
makes a small modification to the sender algorithm so that it won't ask
the receiver to relocate blocks from earlier in the file when running
with the --inplace option.
I would appreciate any testing and feedback people can provide! I
don't expect this to be accepted yet, but I'd like to have what I've
done reviewed for any glaring errors, poor file handling, etc.
So far, on files of about 500M in size, I am noticing a 30-40%
speedup in transfer times.
The patch is available at http://www.wunit.net/rsync-inplace.patch,
and is included below (I apologise for the size of the e-mail).
Thanks,
Mark.
diff -Naur rsync-2.6.1pre2/match.c rsync-inplace/match.c
--- rsync-2.6.1pre2/match.c 2004-01-04 06:28:03.000000000 +1100
+++ rsync-inplace/match.c 2004-04-21 11:04:31.000000000 +1000
@@ -22,6 +22,7 @@
extern int verbose;
extern int am_server;
extern int do_progress;
+extern int inplace;
typedef unsigned short tag;
@@ -199,6 +200,10 @@
if (l != s->sums[i].len)
continue;
+ /* if inplace, make sure the offset is greater than
where we are */
+ if (inplace && (offset > s->sums[i].offset))
+ continue;
+
if (verbose > 3)
rprintf(FINFO,"potential match at %.0f
target=%.0f %.0f sum=%08x\n",
(double)offset,(double)j,(double)i,sum);
diff -Naur rsync-2.6.1pre2/options.c rsync-inplace/options.c
--- rsync-2.6.1pre2/options.c 2004-04-18 03:07:23.000000000 +1000
+++ rsync-inplace/options.c 2004-04-21 11:45:27.000000000 +1000
@@ -91,6 +91,7 @@
int modify_window = 0;
int blocking_io = -1;
int checksum_seed = 0;
+int inplace = 0;
unsigned int block_size = 0;
@@ -231,6 +232,7 @@
rprintf(F," --backup-dir make backups into this
directory\n");
rprintf(F," --suffix=SUFFIX backup suffix (default %s w/o
--backup-dir)\n",BACKUP_SUFFIX);
rprintf(F," -u, --update update only (don't overwrite
newer files)\n");
+ rprintf(F," --inplace update the destination file
inplace *SEE MAN PAGE*\n");
rprintf(F," -l, --links copy symlinks as
symlinks\n");
rprintf(F," -L, --copy-links copy the referent of all
symlinks\n");
rprintf(F," --copy-unsafe-links copy the referent of
\"unsafe\" symlinks\n");
@@ -321,6 +323,7 @@
{"delete", 0, POPT_ARG_NONE, &delete_mode, 0, 0, 0
},
{"existing", 0, POPT_ARG_NONE, &only_existing, 0, 0,
0 },
{"ignore-existing", 0, POPT_ARG_NONE, &opt_ignore_existing,
0, 0,
0 },
+ {"inplace", 0, POPT_ARG_NONE, &inplace, 0, 0, 0 },
{"delete-after", 0, POPT_ARG_NONE, 0,
OPT_DELETE_AFTER, 0, 0 },
{"delete-excluded", 0, POPT_ARG_NONE, 0,
OPT_DELETE_EXCLUDED, 0, 0 },
{"force", 0, POPT_ARG_NONE, &force_delete, 0, 0,
0 },
@@ -912,7 +915,7 @@
args[ac++] = arg;
}
- if (keep_partial)
+ if (keep_partial && !inplace)
args[ac++] = "--partial";
if (force_delete)
@@ -939,6 +942,11 @@
if (opt_ignore_existing && am_sender)
args[ac++] = "--ignore-existing";
+ if (inplace) {
+ keep_partial = 0;
+ args[ac++] = "--inplace";
+ }
+
if (tmpdir) {
args[ac++] = "--temp-dir";
args[ac++] = tmpdir;
diff -Naur rsync-2.6.1pre2/receiver.c rsync-inplace/receiver.c
--- rsync-2.6.1pre2/receiver.c 2004-03-24 03:50:40.000000000 +1100
+++ rsync-inplace/receiver.c 2004-04-22 09:16:47.000000000 +1000
@@ -45,6 +45,7 @@
extern int module_id;
extern int ignore_errors;
extern int orig_umask;
+extern int inplace;
static void delete_one(char *fn, int is_dir)
{
@@ -226,6 +227,7 @@
full_fname(fname), strerror(errno));
exit_cleanup(RERR_FILEIO);
}
+
offset += i;
continue;
}
@@ -249,16 +251,28 @@
sum_update(map,len);
}
- if (fd != -1 && write_file(fd,map,len) != (int) len) {
- rprintf(FERROR, "write failed on %s: %s\n",
- full_fname(fname), strerror(errno));
- exit_cleanup(RERR_FILEIO);
+ if (!inplace || (offset != offset2)) {
+ if (fd != -1 && write_file(fd,map,len) != (int) len) {
+ rprintf(FERROR, "write failed on %s: %s\n",
+ full_fname(fname), strerror(errno));
+ exit_cleanup(RERR_FILEIO);
+ }
+ } else {
+ flush_write_file(fd);
+ if (do_lseek(fd,(OFF_T)len,SEEK_CUR) != (offset+len)) {
+ rprintf(FERROR, "lseek failed on %s: %s, %lli,
%lli, %i\n",
+ full_fname(fname), strerror(errno),
do_lseek(fd,0,SEEK_CUR), (offset+len), i);
+ exit_cleanup(RERR_FILEIO);
+ }
}
offset += len;
}
flush_write_file(fd);
+ if (inplace)
+ ftruncate(fd, offset);
+
if (do_progress)
end_progress(total_size);
@@ -410,39 +424,52 @@
} else
mapbuf = NULL;
- if (!get_tmpname(fnametmp,fname)) {
- if (mapbuf) unmap_file(mapbuf);
- if (fd1 != -1) close(fd1);
- continue;
- }
-
- strlcpy(template, fnametmp, sizeof template);
+ /* We now check to see if we are writing file
"inplace" */
+ if (inplace) {
+ fd2 = do_open(fnamecmp, O_WRONLY|O_CREAT, 0);
+ if (fd2 == -1) {
+ rprintf(FERROR, "open %s failed: %s\n",
+ full_fname(fnametmp), strerror(errno));
+ receive_data(f_in,mapbuf,-1,NULL,file->length);
+ if (mapbuf) unmap_file(mapbuf);
+ if (fd1 != -1) close(fd1);
+ continue;
+ }
+ } else {
+ if (!get_tmpname(fnametmp,fname)) {
+ if (mapbuf) unmap_file(mapbuf);
+ if (fd1 != -1) close(fd1);
+ continue;
+ }
- /* we initially set the perms without the
- * setuid/setgid bits to ensure that there is no race
- * condition. They are then correctly updated after
- * the lchown. Thanks to snabb@epipe.fi for pointing
- * this out. We also set it initially without group
- * access because of a similar race condition. */
- fd2 = do_mkstemp(fnametmp, file->mode & INITACCESSPERMS);
-
- /* in most cases parent directories will already exist
- * because their information should have been previously
- * transferred, but that may not be the case with -R */
- if (fd2 == -1 && relative_paths && errno ==
ENOENT &&
- create_directory_path(fnametmp, orig_umask) == 0) {
- strlcpy(fnametmp, template, sizeof fnametmp);
+ strlcpy(template, fnametmp, sizeof template);
+
+ /* we initially set the perms without the
+ * setuid/setgid bits to ensure that there is no race
+ * condition. They are then correctly updated after
+ * the lchown. Thanks to snabb@epipe.fi for pointing
+ * this out. We also set it initially without group
+ * access because of a similar race condition. */
fd2 = do_mkstemp(fnametmp, file->mode &
INITACCESSPERMS);
+
+ /* in most cases parent directories will already exist
+ * because their information should have been previously
+ * transferred, but that may not be the case with -R */
+ if (fd2 == -1 && relative_paths && errno ==
ENOENT &&
+ create_directory_path(fnametmp, orig_umask) == 0) {
+ strlcpy(fnametmp, template, sizeof fnametmp);
+ fd2 = do_mkstemp(fnametmp, file->mode &
INITACCESSPERMS);
+ }
+ if (fd2 == -1) {
+ rprintf(FERROR, "mkstemp %s failed: %s\n",
+ full_fname(fnametmp), strerror(errno));
+ receive_data(f_in,mapbuf,-1,NULL,file->length);
+ if (mapbuf) unmap_file(mapbuf);
+ if (fd1 != -1) close(fd1);
+ continue;
+ }
}
- if (fd2 == -1) {
- rprintf(FERROR, "mkstemp %s failed: %s\n",
- full_fname(fnametmp), strerror(errno));
- receive_data(f_in,mapbuf,-1,NULL,file->length);
- if (mapbuf) unmap_file(mapbuf);
- if (fd1 != -1) close(fd1);
- continue;
- }
-
+
cleanup_set(fnametmp, fname, file, mapbuf, fd1, fd2);
if (!am_server && verbose) { /* log transfer */
diff -Naur rsync-2.6.1pre2/rsync.1 rsync-inplace/rsync.1
--- rsync-2.6.1pre2/rsync.1 2004-04-18 04:40:16.000000000 +1000
+++ rsync-inplace/rsync.1 2004-04-28 08:02:51.000000000 +1000
@@ -326,6 +326,7 @@
--backup-dir make backups into this directory
--suffix=SUFFIX backup suffix (default ~ w/o --backup-dir)
-u, --update update only (don\&'t overwrite newer
files)
+ --inplace update the destination file inplace
-l, --links copy symlinks as symlinks
-L, --copy-links copy the referent of all symlinks
--copy-unsafe-links copy the referent of "unsafe" symlinks
@@ -537,6 +538,13 @@
destination file already exists and has a date later than the source
file\&.
.IP
+.IP "\fB--inplace\fP"
+This causes rsync not to create a new copy of the file and then move it
into place. Instead rsync will overwrite the existing file, meaning
that the rsync algorithm can't extract the full ammount of network
reduction it might otherwise.\&
+.IP
+This option is useful for transfer of large files with block based
changes and also on systems that are disk bound not network bound.\&
+.IP
+WARNING: If the transfer is interrupted, you will have an inconsistent
file and the transfer should be run again.\&
+.IP
.IP "\fB-l, --links\fP"
When symlinks are encountered, recreate the
symlink on the destination\&.
diff -Naur rsync-2.6.1pre2/rsync.c rsync-inplace/rsync.c
--- rsync-2.6.1pre2/rsync.c 2004-03-24 03:16:15.000000000 +1100
+++ rsync-inplace/rsync.c 2004-04-22 09:16:49.000000000 +1000
@@ -33,6 +33,7 @@
extern int preserve_gid;
extern int preserve_perms;
extern int make_backups;
+extern int inplace;
/*
@@ -235,6 +236,11 @@
if (make_backups && !make_backup(fname))
return;
+ if (inplace) {
+ set_perms(fname,file,NULL,0);
+ return;
+ }
+
/* move tmp file over real file */
ret = robust_rename(fnametmp, fname, file->mode &
INITACCESSPERMS);
if (ret < 0) {