Richard W.M. Jones
2010-Sep-20 17:35 UTC
[Libguestfs] [PATCH 0/2] Implement upload-offset, download-offset APIs, and hexedit guestfish command
This two-part patch implements the 'hexedit' command described here: http://rwmj.wordpress.com/2010/09/18/new-feature-guestfish-hexedit/#content The first part implements two new APIs for partial writing and reading of files. These are modelled after guestfs_upload and guestfs_download but allow you to specify an offset and (for download only) a size: int guestfs_download_offset (guestfs_h *g, const char *remotefilename, const char *filename, int64_t offset, int64_t size); int guestfs_upload_offset (guestfs_h *g, const char *filename, const char *remotefilename, int64_t offset); Since these calls use the streaming FileIn/FileOut part of the protocol, they are both efficient and unlimited in the amount and type of data that can be streamed. http://libguestfs.org/guestfs.3.html#functions_that_have_filein_parameters I also wrote the hexedit command using the putative hread/hwrite API and the code was much longer and more involved. Although there is an advantage to hread/hwrite which is you can read and write to and from memory. The second part of the patch implements the hexedit command itself in guestfish. Rich. -- Richard Jones, Virtualization Group, Red Hat http://people.redhat.com/~rjones New in Fedora 11: Fedora Windows cross-compiler. Compile Windows programs, test, and build Windows installers. Over 70 libraries supprt'd http://fedoraproject.org/wiki/MinGW http://www.annexia.org/fedora_mingw
Richard W.M. Jones
2010-Sep-20 17:36 UTC
[Libguestfs] [PATCH 1/2] New APIs: upload-offset and download-offset
-- Richard Jones, Virtualization Group, Red Hat http://people.redhat.com/~rjones virt-df lists disk usage of guests without needing to install any software inside the virtual machine. Supports Linux and Windows. http://et.redhat.com/~rjones/virt-df/ -------------- next part -------------->From f52395b74eb99e8a5fbda122979a2db53213f511 Mon Sep 17 00:00:00 2001From: Richard W.M. Jones <rjones at redhat.com> Date: Mon, 20 Sep 2010 18:23:58 +0100 Subject: [PATCH 1/2] New APIs: upload-offset and download-offset These APIs allow you to efficiently write and read parts of files or devices. --- daemon/upload.c | 115 ++++++++++++++++++++++++++++++++++++++- generator/generator_actions.ml | 48 +++++++++++++++++ src/MAX_PROC_NR | 2 +- 3 files changed, 161 insertions(+), 4 deletions(-) diff --git a/daemon/upload.c b/daemon/upload.c index 604e705..623ad3a 100644 --- a/daemon/upload.c +++ b/daemon/upload.c @@ -39,15 +39,15 @@ write_cb (void *fd_ptr, const void *buf, size_t len) } /* Has one FileIn parameter. */ -int -do_upload (const char *filename) +static int +upload (const char *filename, int flags, int64_t offset) { int err, fd, r, is_dev; is_dev = STRPREFIX (filename, "/dev/"); if (!is_dev) CHROOT_IN; - fd = open (filename, O_WRONLY|O_CREAT|O_TRUNC|O_NOCTTY, 0666); + fd = open (filename, flags, 0666); if (!is_dev) CHROOT_OUT; if (fd == -1) { err = errno; @@ -57,6 +57,16 @@ do_upload (const char *filename) return -1; } + if (offset) { + if (lseek (fd, offset, SEEK_SET) == -1) { + err = errno; + r = cancel_receive (); + errno = err; + if (r != -2) reply_with_perror ("lseek: %s", filename); + return -1; + } + } + r = receive_file (write_cb, &fd); if (r == -1) { /* write error */ err = errno; @@ -85,6 +95,25 @@ do_upload (const char *filename) return 0; } +/* Has one FileIn parameter. */ +int +do_upload (const char *filename) +{ + return upload (filename, O_WRONLY|O_CREAT|O_TRUNC|O_NOCTTY, 0); +} + +/* Has one FileIn parameter. */ +int +do_upload_offset (const char *filename, int64_t offset) +{ + if (offset < 0) { + reply_with_perror ("%s: offset in file is negative", filename); + return -1; + } + + return upload (filename, O_WRONLY|O_CREAT|O_NOCTTY, offset); +} + /* Has one FileOut parameter. */ int do_download (const char *filename) @@ -156,3 +185,83 @@ do_download (const char *filename) return 0; } + +/* Has one FileOut parameter. */ +int +do_download_offset (const char *filename, int64_t offset, int64_t size) +{ + int fd, r, is_dev; + char buf[GUESTFS_MAX_CHUNK_SIZE]; + + if (offset < 0) { + reply_with_perror ("%s: offset in file is negative", filename); + return -1; + } + + if (size < 0) { + reply_with_perror ("%s: size is negative", filename); + return -1; + } + uint64_t usize = (uint64_t) size; + + is_dev = STRPREFIX (filename, "/dev/"); + + if (!is_dev) CHROOT_IN; + fd = open (filename, O_RDONLY); + if (!is_dev) CHROOT_OUT; + if (fd == -1) { + reply_with_perror ("%s", filename); + return -1; + } + + if (offset) { + if (lseek (fd, offset, SEEK_SET) == -1) { + reply_with_perror ("lseek: %s", filename); + return -1; + } + } + + uint64_t total = usize, sent = 0; + + /* Now we must send the reply message, before the file contents. After + * this there is no opportunity in the protocol to send any error + * message back. Instead we can only cancel the transfer. + */ + reply (NULL, NULL); + + while (usize > 0) { + r = read (fd, buf, usize > sizeof buf ? sizeof buf : usize); + if (r == -1) { + perror (filename); + send_file_end (1); /* Cancel. */ + close (fd); + return -1; + } + + if (r == 0) + /* The documentation leaves this case undefined. Currently we + * just read fewer bytes than requested. + */ + break; + + if (send_file_write (buf, r) < 0) { + close (fd); + return -1; + } + + sent += r; + usize -= r; + notify_progress (sent, total); + } + + if (close (fd) == -1) { + perror (filename); + send_file_end (1); /* Cancel. */ + return -1; + } + + if (send_file_end (0)) /* Normal end of file. */ + return -1; + + return 0; +} diff --git a/generator/generator_actions.ml b/generator/generator_actions.ml index d01871f..7711d9c 100644 --- a/generator/generator_actions.ml +++ b/generator/generator_actions.ml @@ -5117,6 +5117,54 @@ removes the partition number, returning the device name The named partition must exist, for example as a string returned from C<guestfs_list_partitions>."); + ("upload_offset", (RErr, [FileIn "filename"; Dev_or_Path "remotefilename"; Int64 "offset"]), 273, [], + (let md5 = Digest.to_hex (Digest.file "COPYING.LIB") in + [InitBasicFS, Always, TestOutput ( + [["upload_offset"; "../COPYING.LIB"; "/COPYING.LIB"; "0"]; + ["checksum"; "md5"; "/COPYING.LIB"]], md5)]), + "upload a file from the local machine with offset", + "\ +Upload local file C<filename> to C<remotefilename> on the +filesystem. + +C<remotefilename> is overwritten starting at the byte C<offset> +specified. The intention is to overwrite parts of existing +files or devices, although if a non-existant file is specified +then it is created with a \"hole\" before C<offset>. The +size of the data written is implicit in the size of the +source C<filename>. + +Note that there is no limit on the amount of data that +can be uploaded with this call, unlike with C<guestfs_pwrite>, +and this call always writes the full amount unless an +error occurs. + +See also C<guestfs_upload>."); + + ("download_offset", (RErr, [Dev_or_Path "remotefilename"; FileOut "filename"; Int64 "offset"; Int64 "size"]), 274, [Progress], + (let md5 = Digest.to_hex (Digest.file "COPYING.LIB") in + let size = string_of_int (Unix.stat "COPYING.LIB").Unix.st_size in + [InitBasicFS, Always, TestOutput ( + (* Pick a file from cwd which isn't likely to change. *) + [["upload"; "../COPYING.LIB"; "/COPYING.LIB"]; + ["download_offset"; "/COPYING.LIB"; "testdownload.tmp"; "100"; size]; + ["upload_offset"; "testdownload.tmp"; "/upload"; "100"]; + ["checksum"; "md5"; "/upload"]], md5)]), + "download a file to the local machine with offset and size", + "\ +Download file C<remotefilename> and save it as C<filename> +on the local machine. + +C<remotefilename> is read for C<size> bytes starting at C<offset> +(this region must be within the file or device). + +Note that there is no limit on the amount of data that +can be downloaded with this call, unlike with C<guestfs_pread>, +and this call always reads the full amount unless an +error occurs. + +See also C<guestfs_download>, C<guestfs_pread>."); + ] let all_functions = non_daemon_functions @ daemon_functions diff --git a/src/MAX_PROC_NR b/src/MAX_PROC_NR index 31e9cf9..d4d5a4b 100644 --- a/src/MAX_PROC_NR +++ b/src/MAX_PROC_NR @@ -1 +1 @@ -272 +274 -- 1.7.2.3
Richard W.M. Jones
2010-Sep-20 17:37 UTC
[Libguestfs] [PATCH 2/2] fish: Implement 'hexedit' command.
-- Richard Jones, Virtualization Group, Red Hat http://people.redhat.com/~rjones libguestfs lets you edit virtual machines. Supports shell scripting, bindings from many languages. http://et.redhat.com/~rjones/libguestfs/ See what it can do: http://et.redhat.com/~rjones/libguestfs/recipes.html -------------- next part -------------->From 9c348a3dc0ddc5a7f404bf8ab719fa704c1b8504 Mon Sep 17 00:00:00 2001From: Richard W.M. Jones <rjones at redhat.com> Date: Sat, 18 Sep 2010 10:09:04 +0100 Subject: [PATCH 2/2] fish: Implement 'hexedit' command. --- fish/Makefile.am | 1 + fish/fish.h | 3 + fish/guestfish.pod | 9 ++- fish/hexedit.c | 190 ++++++++++++++++++++++++++++++++++++++++ generator/generator_actions.ml | 36 ++++++++ po/POTFILES.in | 1 + 6 files changed, 239 insertions(+), 1 deletions(-) create mode 100644 fish/hexedit.c diff --git a/fish/Makefile.am b/fish/Makefile.am index 4d17aa1..91094df 100644 --- a/fish/Makefile.am +++ b/fish/Makefile.am @@ -48,6 +48,7 @@ guestfish_SOURCES = \ fish.c \ fish.h \ glob.c \ + hexedit.c \ inspect.c \ lcd.c \ man.c \ diff --git a/fish/fish.h b/fish/fish.h index 609cbc2..220a36a 100644 --- a/fish/fish.h +++ b/fish/fish.h @@ -104,6 +104,9 @@ extern int run_echo (const char *cmd, int argc, char *argv[]); /* in edit.c */ extern int run_edit (const char *cmd, int argc, char *argv[]); +/* in hexedit.c */ +extern int run_hexedit (const char *cmd, int argc, char *argv[]); + /* in inspect.c */ extern void inspect_mount (void); extern void print_inspect_prompt (void); diff --git a/fish/guestfish.pod b/fish/guestfish.pod index ed2e798..7f914fb 100644 --- a/fish/guestfish.pod +++ b/fish/guestfish.pod @@ -808,6 +808,12 @@ Used with the I<--remote> option to specify the remote guestfish process to control. See section L</REMOTE CONTROL GUESTFISH OVER A SOCKET>. +=item HEXEDITOR + +The L</hexedit> command uses C<$HEXEDITOR> as the external hex +editor. If not specified, the external L<hexedit(1)> program +is used. + =item HOME If compiled with GNU readline support, various files in the @@ -925,7 +931,8 @@ L<virt-make-fs(1)>, L<virt-rescue(1)>, L<virt-resize(1)>, L<virt-tar(1)>, -L<virt-win-reg(1)>. +L<virt-win-reg(1)>, +L<hexedit(1)>. =head1 AUTHORS diff --git a/fish/hexedit.c b/fish/hexedit.c new file mode 100644 index 0000000..69a9ef8 --- /dev/null +++ b/fish/hexedit.c @@ -0,0 +1,190 @@ +/* guestfish - the filesystem interactive shell + * Copyright (C) 2010 Red Hat Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. + */ + +#include <config.h> + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <unistd.h> +#include <fcntl.h> +#include <inttypes.h> +#include <sys/types.h> +#include <sys/stat.h> + +#include "fish.h" + +#define MAX_DOWNLOAD_SIZE (16 * 1024 * 1024) +#define MAX_DOWNLOAD_SIZE_TEXT "16MB" + +static off_t get_size (const char *filename); + +int +run_hexedit (const char *cmd, int argc, char *argv[]) +{ + if (argc < 1 || argc > 3) { + fprintf (stderr, _("hexedit (device|filename) [max | start max]\n")); + return -1; + } + + const char *filename = argv[0]; + off_t size = get_size (filename); + if (size == -1) + return -1; + + if (size == 0) { + fprintf (stderr, + _("hexedit: %s is a zero length file or device\n"), filename); + return -1; + } + + off_t start; + off_t max; + + if (argc == 1) { /* hexedit device */ + /* Check we're not going to download a huge file. */ + if (size > MAX_DOWNLOAD_SIZE) { + fprintf (stderr, + _("hexedit: %s is larger than %s. You must supply a limit using\n" + " 'hexedit %s <max>' (eg. 'hexedit %s 1M') or a range using\n" + " 'hexedit %s <start> <max>'.\n"), + filename, MAX_DOWNLOAD_SIZE_TEXT, + filename, filename, + filename); + return -1; + } + + start = 0; + max = size; + } + else { + if (argc == 3) { /* hexedit device start max */ + if (parse_size (argv[1], &start) == -1) + return -1; + if (parse_size (argv[2], &max) == -1) + return -1; + } else { /* hexedit device max */ + start = 0; + if (parse_size (argv[1], &max) == -1) + return -1; + } + + if (start + max > size) + max = size - start; + } + + if (max <= 0) { + fprintf (stderr, _("hexedit: invalid range\n")); + return -1; + } + + /* Download the requested range from the remote file|device into a + * local temporary file. + */ + const char *editor; + int r; + struct stat oldstat, newstat; + char buf[BUFSIZ]; + char tmp[] = "/tmp/guestfishXXXXXX"; + int fd = mkstemp (tmp); + if (fd == -1) { + perror ("mkstemp"); + return -1; + } + + /* Choose an editor. */ + editor = getenv ("HEXEDITOR"); + if (editor == NULL) + editor = "hexedit"; + + snprintf (buf, sizeof buf, "/dev/fd/%d", fd); + + if (guestfs_download_offset (g, filename, tmp, start, max) == -1) { + unlink (tmp); + close (fd); + return -1; + } + + if (close (fd) == -1) { + unlink (tmp); + return -1; + } + + /* Get the old stat. */ + if (stat (tmp, &oldstat) == -1) { + perror (tmp); + unlink (tmp); + return -1; + } + + /* Edit it. */ + snprintf (buf, sizeof buf, "%s %s", editor, tmp); + + r = system (buf); + if (r != 0) { + perror (buf); + unlink (tmp); + return -1; + } + + /* Get the new stat. */ + if (stat (tmp, &newstat) == -1) { + perror (tmp); + unlink (tmp); + return -1; + } + + /* Changed? */ + if (oldstat.st_ctime == newstat.st_ctime && + oldstat.st_size == newstat.st_size) { + unlink (tmp); + return 0; + } + + /* Write new content. */ + if (guestfs_upload_offset (g, tmp, filename, start) == -1) { + unlink (tmp); + return -1; + } + + unlink (tmp); + return 0; +} + +/* Get the size of the file or block device. */ +static off_t +get_size (const char *filename) +{ + int64_t size; + + if (STRPREFIX (filename, "/dev/")) { + size = guestfs_blockdev_getsize64 (g, filename); + if (size == -1) + return -1; + } + else { + size = guestfs_filesize (g, filename); + if (size == -1) + return -1; + } + + /* This case should be safe because we always compile with + * 64 bit file offsets. + */ + return (off_t) size; +} diff --git a/generator/generator_actions.ml b/generator/generator_actions.ml index 7711d9c..bbbb4c4 100644 --- a/generator/generator_actions.ml +++ b/generator/generator_actions.ml @@ -5269,6 +5269,42 @@ repeatedly on each matching path. See L</WILDCARDS AND GLOBBING>."); + ("hexedit", (RErr,[]), -1, [], [], + "edit with a hex editor", + " hexedit <filename|device> + hexedit <filename|device> <max> + hexedit <filename|device> <start> <max> + +Use hexedit (a hex editor) to edit all or part of a binary file +or block device. + +This command works by downloading potentially the whole file or +device, editing it locally, then uploading it. If the file or +device is large, you have to specify which part you wish to edit +by using C<max> and/or C<start> C<max> parameters. +C<start> and C<max> are specified in bytes, with the usual +modifiers allowed such as C<1M> (1 megabyte). + +For example to edit the first few sectors of a disk you +might do: + + hexedit /dev/sda 1M + +which would allow you to edit anywhere within the first megabyte +of the disk. + +To edit the superblock of an ext2 filesystem on C</dev/sda1>, do: + + hexedit /dev/sda1 0x400 0x400 + +(assuming the superblock is in the standard location). + +This command requires the external L<hexedit(1)> program. You +can specify another program to use by setting the C<HEXEDITOR> +environment variable. + +See also L</hexdump>."); + ("lcd", (RErr,[]), -1, [], [], "change working directory", " lcd directory diff --git a/po/POTFILES.in b/po/POTFILES.in index 4518eab..10b690c 100644 --- a/po/POTFILES.in +++ b/po/POTFILES.in @@ -78,6 +78,7 @@ fish/echo.c fish/edit.c fish/fish.c fish/glob.c +fish/hexedit.c fish/inspect.c fish/lcd.c fish/man.c -- 1.7.2.3