This is a patch to add an --rsync-exclude option to rsync-2.5.6cvs.
File names in .rsync- (or .rsync+) are excluded (or included) from the file
lists associated with the current directory and all of its subdirectories.
This has advantages over --cvs-exclude for backing up large file systems
since the .cvsignore files only apply to the current directory:
unless the .cvsignore restrictions apply to the entire tree they must be
duplicated in each subdirectory. In any case, the --cvs-exclude option
isn't intended for general system backups (for example, unless the default
list is cleared with "!", it automatically excludes *.a libraries).
The .rsync-/.rsync+ mechanism provided by this simple patch provides a more
general solution, since files may be added to or removed from the exclude
list at all levels in the directory hierarchy (similar to apache .htaccess
files).
This patch also adds an example for using --link-dest to the man page:
"This command creates a linked backup of /home named current on nimbus
relative to a previous backup named previous:
rsync -vax --delete --numeric-ids --link-dest=previous /home
nimbus:current"
diff -ru rsync-2.5.6cvs/flist.c rsync-2.5.6cvsJ/flist.c
--- rsync-2.5.6cvs/flist.c Tue Dec 24 00:42:04 2002
+++ rsync-2.5.6cvsJ/flist.c Sat Jan 11 03:47:14 2003
@@ -39,6 +39,7 @@
extern int always_checksum;
extern int cvs_exclude;
+extern int rsync_exclude;
extern int recurse;
@@ -64,6 +65,7 @@
static char topsrcname[MAXPATHLEN];
static struct exclude_struct **local_exclude_list;
+static struct exclude_struct **recur_local_exclude_list;
static struct file_struct null_file;
@@ -260,6 +262,9 @@
if (check_exclude(fname, local_exclude_list, st)) {
return 1;
}
+ if (check_exclude(fname, recur_local_exclude_list, st)) {
+ return 1;
+ }
return 0;
}
@@ -799,8 +804,11 @@
if (S_ISDIR(file->mode) && recursive) {
struct exclude_struct **last_exclude_list local_exclude_list;
+ struct exclude_struct **recur_last_exclude_list +
recur_local_exclude_list;
send_directory(f, flist, f_name(file));
local_exclude_list = last_exclude_list;
+ recur_local_exclude_list = recur_last_exclude_list;
return;
}
}
@@ -840,6 +848,29 @@
local_exclude_list = NULL;
+ if (rsync_exclude) {
+ if (strlen(fname) + strlen(".rsync-") <= MAXPATHLEN - 1) {
+ strcpy(p, ".rsync-");
+ recur_local_exclude_list + make_exclude_list(fname,
recur_local_exclude_list, 0, 0);
+ } else {
+ io_error = 1;
+ rprintf(FINFO,
+ "cannot cvs-exclude in long-named directory %s\n",
+ fname);
+ }
+ if (strlen(fname) + strlen(".rsync+") <= MAXPATHLEN - 1) {
+ strcpy(p, ".rsync+");
+ recur_local_exclude_list + make_exclude_list(fname,
recur_local_exclude_list, 0, 1);
+ } else {
+ io_error = 1;
+ rprintf(FINFO,
+ "cannot cvs-exclude in long-named directory %s\n",
+ fname);
+ }
+ }
+
if (cvs_exclude) {
if (strlen(fname) + strlen(".cvsignore") <= MAXPATHLEN - 1) {
strcpy(p, ".cvsignore");
diff -ru rsync-2.5.6cvs/options.c rsync-2.5.6cvsJ/options.c
--- rsync-2.5.6cvs/options.c Tue Jan 14 09:35:25 2003
+++ rsync-2.5.6cvsJ/options.c Tue Jan 14 09:35:40 2003
@@ -45,6 +45,7 @@
int preserve_times = 0;
int update_only = 0;
int cvs_exclude = 0;
+int rsync_exclude = 0;
int dry_run=0;
int local_server=0;
int ignore_times=0;
@@ -237,6 +238,7 @@
rprintf(F," -e, --rsh=COMMAND specify the remote
shell\n");
rprintf(F," --rsync-path=PATH specify path to rsync on the
remote machine\n");
rprintf(F," -C, --cvs-exclude auto ignore files in the same
way CVS does\n");
+ rprintf(F," --rsync-exclude exclude (include) files listed
in .rsync- (.rsync+)\n");
rprintf(F," --existing only update files that already
exist\n");
rprintf(F," --ignore-existing ignore files that already exist
on the receiving side\n");
rprintf(F," --delete delete files that don't
exist on the sending side\n");
@@ -324,6 +326,7 @@
{"dry-run", 'n', POPT_ARG_NONE, &dry_run , 0,
0, 0 },
{"sparse", 'S', POPT_ARG_NONE, &sparse_files
, 0, 0, 0 },
{"cvs-exclude", 'C', POPT_ARG_NONE, &cvs_exclude
, 0, 0, 0 },
+ {"rsync-exclude", 'C', POPT_ARG_NONE,
&rsync_exclude , 0, 0, 0 },
{"update", 'u', POPT_ARG_NONE, &update_only
, 0, 0, 0 },
{"links", 'l', POPT_ARG_NONE,
&preserve_links , 0, 0, 0 },
{"copy-links", 'L', POPT_ARG_NONE, ©_links ,
0, 0, 0 },
diff -ru rsync-2.5.6cvs/rsync.yo rsync-2.5.6cvsJ/rsync.yo
--- rsync-2.5.6cvs/rsync.yo Tue Jan 14 09:35:25 2003
+++ rsync-2.5.6cvsJ/rsync.yo Tue Jan 14 09:35:40 2003
@@ -276,6 +276,11 @@
this is launched from cron every few hours.
+This command creates a linked backup of /home named current on nimbus
+relative to a previous backup named previous:
+
+rsync -vax --delete --numeric-ids --link-dest=previous /home nimbus:current
+
manpagesection(OPTIONS SUMMARY)
Here is a short summary of the options available in rsync. Please refer
@@ -311,6 +316,7 @@
-e, --rsh=COMMAND specify the remote shell to use
--rsync-path=PATH specify path to rsync on the remote machine
-C, --cvs-exclude auto ignore files in the same way CVS does
+ --rsync-exclude exclude (include) files listed in .rsync-
(.rsync+)
--existing only update files that already exist
--ignore-existing ignore files that already exist on the receiving
side
--delete delete files that don't exist on the sending
side
@@ -654,6 +660,10 @@
.cvsignore file and matches one of the patterns listed therein. See
the bf(cvs(1)) manual for more information.
+dit(bf(--rsync-exclude)) File names in .rsync- (or .rsync+) are excluded
+(or included) from the file lists associated with the current directory and
+all of its subdirectories.
+
dit(bf(--csum-length=LENGTH)) By default the primary checksum used in
rsync is a very strong 16 byte MD4 checksum. In most cases you will
find that a truncated version of this checksum is quite efficient, and
On Tue, Jan 14, 2003 at 09:48:31AM -0700, John Bowman wrote:> This is a patch to add an --rsync-exclude option to rsync-2.5.6cvs. > File names in .rsync- (or .rsync+) are excluded (or included) from the file > lists associated with the current directory and all of its subdirectories. > > This has advantages over --cvs-exclude for backing up large file systems > since the .cvsignore files only apply to the current directory: > unless the .cvsignore restrictions apply to the entire tree they must be > duplicated in each subdirectory. In any case, the --cvs-exclude option > isn't intended for general system backups (for example, unless the default > list is cleared with "!", it automatically excludes *.a libraries). > > The .rsync-/.rsync+ mechanism provided by this simple patch provides a more > general solution, since files may be added to or removed from the exclude > list at all levels in the directory hierarchy (similar to apache .htaccess > files).An interesting idea. Would be worthwhile putting in the patches directory and solicit feedback to see how much use it gets. You also bring up a good point about the --cvs-exclude option. In excluding *.a and *.so it limits its utility. This should be noted in the manpage. One wonders if perhaps a subset of --cvs-exclude's fixed patterns might be worth having as an --bk-exclude option. Perhaps *~ #* .#* ,* *.old *.bak *.BAK core Purely as a convenience.> This patch also adds an example for using --link-dest to the man page: > > "This command creates a linked backup of /home named current on nimbus > relative to a previous backup named previous: > > rsync -vax --delete --numeric-ids --link-dest=previous /home nimbus:current"I don't know if that is needed. Lacking context the example has minimal meaning and compare-dest doesn't have an example either. Should be a separate patch. -- ________________________________________________________________ J.W. Schultz Pegasystems Technologies email address: jw@pegasys.ws Remember Cernan and Schmitt
>> This patch also adds an example for using --link-dest to the man page: > > I don't know if that is needed. Lacking context the example > has minimal meaning and compare-dest doesn't have an exampleA more complete context is available here: http://www.math.ualberta.ca/imaging/rlbackup/ rlbackup is a simple yet secure backup mechanism for generating linked backups over the network, with historical pruning, thanks to rsync and your very useful --link-dest option. -- John Bowman Associate Professor Department of Mathematical and Statistical Sciences University of Alberta Edmonton, Alberta Canada T6G 2G1 Office: CAB 521 Phone: +1 780 492 0532 FAX: +1 780 492 6826 http://www.math.ualberta.ca/~bowman Email: bowman@math.ualberta.ca
The patch below supercedes my patch dated 14 Jan 2003. The new version
fixes serious problems due to memory reallocation/freeing of lists and is
also more efficient. The scheme has been simplified to store excluded
patterns in .rsync only (a "+ " pattern prefix may be used to force
inclusion).
Note: The precedence in check_exclude has been improved to use the most local
and most recent matching pattern. This is accomplished by searching
(backwards in each file) first the patterns found in .rsync, then in
.cvsignore, and finally in the global --exclude list.
The latest version of this patch is available at
http://www.math.ualberta.ca/imaging/rlbackup/
-- John Bowman
Associate Professor
Department of Mathematical and Statistical Sciences
University of Alberta
Edmonton, Alberta
Canada T6G 2G1
This is a patch to add an --rsync-exclude option to rsync-2.5.6cvs.
Patterns in .rsync can be used to recursively exclude/include files in the
current directory and all of its subdirectories.
This has advantages over --cvs-exclude for backing up large file systems
since the .cvsignore files only apply to the current directory:
unless the .cvsignore restrictions apply to the entire tree they must be
duplicated in each subdirectory. In any case, the --cvs-exclude option
isn't intended for general system backups (for example, unless the default
list is cleared with "!", it automatically excludes *.a and *.so
libraries).
diff -ru rsync-2.5.6cvs/exclude.c rsync-2.5.6cvsJ/exclude.c
--- rsync-2.5.6cvs/exclude.c Tue Jan 14 09:35:25 2003
+++ rsync-2.5.6cvsJ/exclude.c Thu Jan 16 19:08:45 2003
@@ -127,6 +127,7 @@
static void report_exclude_result(char const *name,
struct exclude_struct const *ent,
+ char const *type,
STRUCT_STAT const *st)
{
/* If a trailing slash is present to match only directories,
@@ -134,10 +135,10 @@
* case we add it back in here. */
if (verbose >= 2)
- rprintf(FINFO, "%s %s %s because of pattern %s%s\n",
+ rprintf(FINFO, "%s %s %s because of %s pattern
%s%s\n",
ent->include ? "including" :
"excluding",
S_ISDIR(st->st_mode) ? "directory" :
"file",
- name, ent->pattern,
+ name, type, ent->pattern,
ent->directory ? "/" : "");
}
@@ -147,6 +148,7 @@
* LOCAL_EXCLUDE_LIST or the globals EXCLUDE_LIST.
*/
int check_exclude(char *name, struct exclude_struct **local_exclude_list,
+ struct exclude_struct **recur_local_exclude_list,
STRUCT_STAT *st)
{
int n;
@@ -156,21 +158,37 @@
/* never exclude '.', even if somebody does --exclude '*' */
return 0;
- if (exclude_list) {
- for (n=0; exclude_list[n]; n++) {
- ent = exclude_list[n];
+ /* Precedence: use the most local and most recent matching pattern,
+ in this order: .rsync, .cvsignore, --exclude */
+
+ if (recur_local_exclude_list) {
+ for (n=0; recur_local_exclude_list[n]; n++) ;
+ for (n--; n >= 0; n--) {
+ ent = recur_local_exclude_list[n];
if (check_one_exclude(name, ent, st)) {
- report_exclude_result(name, ent, st);
+ report_exclude_result(name, ent,
"rsync", st);
return !ent->include;
}
}
}
if (local_exclude_list) {
- for (n=0; local_exclude_list[n]; n++) {
+ for (n=0; local_exclude_list[n]; n++) ;
+ for (n--; n >= 0; n--) {
ent = local_exclude_list[n];
if (check_one_exclude(name, ent, st)) {
- report_exclude_result(name, ent, st);
+ report_exclude_result(name, ent,
"cvsignore", st);
+ return !ent->include;
+ }
+ }
+ }
+
+ if (exclude_list) {
+ for (n=0; exclude_list[n]; n++) ;
+ for (n--; n >= 0; n--) {
+ ent = exclude_list[n];
+ if (check_one_exclude(name, ent, st)) {
+ report_exclude_result(name, ent,
"global", st);
return !ent->include;
}
}
diff -ru rsync-2.5.6cvs/flist.c rsync-2.5.6cvsJ/flist.c
--- rsync-2.5.6cvs/flist.c Wed Jan 15 23:09:15 2003
+++ rsync-2.5.6cvsJ/flist.c Thu Jan 16 19:09:00 2003
@@ -39,6 +39,7 @@
extern int always_checksum;
extern int cvs_exclude;
+extern int rsync_exclude;
extern int recurse;
@@ -62,6 +63,7 @@
extern int write_batch;
static struct exclude_struct **local_exclude_list;
+static struct exclude_struct **recur_local_exclude_list;
static struct file_struct null_file;
@@ -258,7 +260,8 @@
if ((f == -1) && delete_excluded) {
return 0;
}
- if (check_exclude(fname, local_exclude_list, st)) {
+ if (check_exclude(fname, local_exclude_list,
+ recur_local_exclude_list, st)) {
return 1;
}
return 0;
@@ -773,7 +776,32 @@
return file;
}
+static struct exclude_struct **copy_exclude_list(struct exclude_struct **from)
{
+ struct exclude_struct **to;
+ int i;
+ int len=0;
+ int size;
+
+ if (!from) return NULL;
+
+ for (; from[len]; len++) ;
+ size=sizeof(struct exclude_struct *)*(len+1);
+ to = (struct exclude_struct **) malloc(size);
+ if (!to) out_of_memory("copy_exclude_list");
+
+ size=sizeof(struct exclude_struct);
+ for (i=0; i < len; i++) {
+ struct exclude_struct *p;
+ p=to[i]=(struct exclude_struct *) malloc(size);
+ if (!p) out_of_memory("copy_exclude_list");
+ *p=*from[i];
+ p->pattern=strdup(from[i]->pattern);
+ if (!p->pattern) out_of_memory("copy_exclude_list");
+ }
+ to[len]=NULL;
+ return to;
+}
void send_file_name(int f, struct file_list *flist, char *fname,
int recursive, unsigned base_flags)
@@ -800,8 +828,11 @@
if (S_ISDIR(file->mode) && recursive) {
struct exclude_struct **last_exclude_list local_exclude_list;
+ struct exclude_struct **recur_last_exclude_list +
recur_local_exclude_list;
send_directory(f, flist, f_name(file));
local_exclude_list = last_exclude_list;
+ recur_local_exclude_list = recur_last_exclude_list;
return;
}
}
@@ -840,6 +871,7 @@
p = fname + strlen(fname);
local_exclude_list = NULL;
+ recur_local_exclude_list = copy_exclude_list(recur_local_exclude_list);
if (cvs_exclude) {
if (strlen(fname) + strlen(".cvsignore") <= MAXPATHLEN - 1) {
@@ -854,6 +886,19 @@
}
}
+ if (rsync_exclude) {
+ if (strlen(fname) + strlen(".rsync") <= MAXPATHLEN - 1) {
+ strcpy(p, ".rsync");
+ recur_local_exclude_list + make_exclude_list(fname,
recur_local_exclude_list, 0, 0);
+ } else {
+ io_error = 1;
+ rprintf(FINFO,
+ "cannot cvs-exclude in long-named directory %s\n",
+ fname);
+ }
+ }
+
for (di = readdir(d); di; di = readdir(d)) {
char *dname = d_name(di);
if (strcmp(dname, ".") == 0 || strcmp(dname, "..") == 0)
@@ -866,6 +911,10 @@
add_exclude_list("!", &local_exclude_list, 0);
}
+ if (recur_local_exclude_list) {
+ add_exclude_list("!", &recur_local_exclude_list, 0);
+ }
+
closedir(d);
}
@@ -887,6 +936,8 @@
if (show_filelist_p() && f != -1)
start_filelist_progress("building file list");
+ recur_local_exclude_list = NULL;
+
start_write = stats.total_written;
flist = flist_new();
diff -ru rsync-2.5.6cvs/options.c rsync-2.5.6cvsJ/options.c
--- rsync-2.5.6cvs/options.c Thu Jan 16 19:17:23 2003
+++ rsync-2.5.6cvsJ/options.c Thu Jan 16 19:14:38 2003
@@ -45,6 +45,7 @@
int preserve_times = 0;
int update_only = 0;
int cvs_exclude = 0;
+int rsync_exclude = 0;
int dry_run=0;
int local_server=0;
int ignore_times=0;
@@ -237,6 +238,7 @@
rprintf(F," -e, --rsh=COMMAND specify the remote
shell\n");
rprintf(F," --rsync-path=PATH specify path to rsync on the
remote machine\n");
rprintf(F," -C, --cvs-exclude auto ignore files in the same
way CVS does\n");
+ rprintf(F," --rsync-exclude recursively exclude files listed
in .rsync\n");
rprintf(F," --existing only update files that already
exist\n");
rprintf(F," --ignore-existing ignore files that already exist
on the receiving side\n");
rprintf(F," --delete delete files that don't
exist on the sending side\n");
@@ -324,6 +326,7 @@
{"dry-run", 'n', POPT_ARG_NONE, &dry_run , 0,
0, 0 },
{"sparse", 'S', POPT_ARG_NONE, &sparse_files
, 0, 0, 0 },
{"cvs-exclude", 'C', POPT_ARG_NONE, &cvs_exclude
, 0, 0, 0 },
+ {"rsync-exclude", 0, POPT_ARG_NONE, &rsync_exclude , 0,
0, 0 },
{"update", 'u', POPT_ARG_NONE, &update_only
, 0, 0, 0 },
{"links", 'l', POPT_ARG_NONE,
&preserve_links , 0, 0, 0 },
{"copy-links", 'L', POPT_ARG_NONE, ©_links ,
0, 0, 0 },
diff -ru rsync-2.5.6cvs/proto.h rsync-2.5.6cvsJ/proto.h
--- rsync-2.5.6cvs/proto.h Wed Jan 15 23:09:15 2003
+++ rsync-2.5.6cvsJ/proto.h Thu Jan 16 11:50:52 2003
@@ -57,6 +57,7 @@
void setup_protocol(int f_out,int f_in);
int claim_connection(char *fname,int max_connections);
int check_exclude(char *name, struct exclude_struct **local_exclude_list,
+ struct exclude_struct **recur_local_exclude_list,
STRUCT_STAT *st);
void add_exclude_list(const char *pattern, struct exclude_struct ***list, int
include);
void add_exclude(const char *pattern, int include);
diff -ru rsync-2.5.6cvs/rsync.yo rsync-2.5.6cvsJ/rsync.yo
--- rsync-2.5.6cvs/rsync.yo Thu Jan 16 19:17:23 2003
+++ rsync-2.5.6cvsJ/rsync.yo Thu Jan 16 19:14:38 2003
@@ -311,6 +316,7 @@
-e, --rsh=COMMAND specify the remote shell to use
--rsync-path=PATH specify path to rsync on the remote machine
-C, --cvs-exclude auto ignore files in the same way CVS does
+ --rsync-exclude recursively exclude files listed in .rsync
--existing only update files that already exist
--ignore-existing ignore files that already exist on the receiving
side
--delete delete files that don't exist on the sending
side
@@ -654,6 +660,12 @@
.cvsignore file and matches one of the patterns listed therein. See
the bf(cvs(1)) manual for more information.
+dit(bf(--rsync-exclude)) File names in .rsync are excluded from the file
+lists associated with the current directory and all of its subdirectories.
+Prefixing the file name with "+ " will force inclusion of the file.
+If there are multiplying matching patterns, the most local and most recent
+matching pattern will be used, in this order: .rsync, .cvsignore, --exclude.
+
dit(bf(--csum-length=LENGTH)) By default the primary checksum used in
rsync is a very strong 16 byte MD4 checksum. In most cases you will
find that a truncated version of this checksum is quite efficient, and