From: Mike Frysinger <vapier at chromium.org>
Add support to the sftp-server for the copy-data extension defined in
[1]. This provides for efficient copying on the server side otherwise
clients have to emulate it by downloading the file before uploading it
to the new path. For large files or slow networks, this can be
prohibitively slow. Now doing a file copy is limited only by the server
I/O.
For clients that provide virtual file systems on top of SFTP, this can
provide significant performance gains.
[1]
https://tools.ietf.org/html/draft-ietf-secsh-filexfer-extensions-00#section-7
---
PROTOCOL | 39 ++++++++++++++++++++++-
sftp-server.c | 87 +++++++++++++++++++++++++++++++++++++++++++++++++++
2 files changed, 125 insertions(+), 1 deletion(-)
diff --git a/PROTOCOL b/PROTOCOL
index f75c1c0ae5b0..04a392db33be 100644
--- a/PROTOCOL
+++ b/PROTOCOL
@@ -459,12 +459,49 @@ This request asks the server to call fsync(2) on an open
file handle.
string "fsync at openssh.com"
string handle
-One receiving this request, a server will call fsync(handle_fd) and will
+On receiving this request, a server will call fsync(handle_fd) and will
respond with a SSH_FXP_STATUS message.
This extension is advertised in the SSH_FXP_VERSION hello with version
"1".
+3.7. sftp: Extension request "copy-data"
+
+This request asks the server to copy data from one open file handle and
+write it to a different open file handle. This avoids needing to
+download the data before uploading it across the network twice.
+
+ byte SSH_FXP_EXTENDED
+ uint32 id
+ string "copy-data"
+ string read-from-handle
+ uint64 read-from-offset
+ uint64 read-data-length
+ string write-to-handle
+ uint64 write-to-offset
+
+The server will copy read-data-length bytes starting from
+read-from-offset from the read-from-handle and write them to
+write-to-handle starting from write-to-offset, and then respond with a
+SSH_FXP_STATUS message.
+
+It's equivalent to issuing a series of SSH_FXP_READ requests on
+read-from-handle and a series of requests of SSH_FXP_WRITE on
+write-to-handle.
+
+If read-from-handle and write-to-handle are the same, the server will
+fail the request and respond with a SSH_FX_INVALID_PARAMETER message.
+
+If read-data-length is 0, then the server will read data from the
+read-from-handle until EOF is reached.
+
+This extension is advertised in the SSH_FXP_VERSION hello with version
+"1".
+
+This request is identical to the "copy-data" request documented in:
+
+https://tools.ietf.org/html/draft-ietf-secsh-filexfer-extensions-00#section-7
+
4. Miscellaneous changes
4.1 Public key format
diff --git a/sftp-server.c b/sftp-server.c
index beac78ed0324..2d7c1c87e83f 100644
--- a/sftp-server.c
+++ b/sftp-server.c
@@ -107,6 +107,7 @@ static void process_extended_statvfs(u_int32_t id);
static void process_extended_fstatvfs(u_int32_t id);
static void process_extended_hardlink(u_int32_t id);
static void process_extended_fsync(u_int32_t id);
+static void process_extended_copy_data(u_int32_t id);
static void process_extended(u_int32_t id);
struct sftp_handler {
@@ -148,6 +149,7 @@ static const struct sftp_handler extended_handlers[] = {
{ "fstatvfs", "fstatvfs at openssh.com", 0,
process_extended_fstatvfs, 0 },
{ "hardlink", "hardlink at openssh.com", 0,
process_extended_hardlink, 1 },
{ "fsync", "fsync at openssh.com", 0,
process_extended_fsync, 1 },
+ { "copy-data", "copy-data", 0, process_extended_copy_data,
1 },
{ NULL, NULL, 0, NULL, 0 }
};
@@ -666,6 +668,9 @@ process_init(void)
(r = sshbuf_put_cstring(msg, "1")) != 0 || /* version */
/* fsync extension */
(r = sshbuf_put_cstring(msg, "fsync at openssh.com")) != 0 ||
+ (r = sshbuf_put_cstring(msg, "1")) != 0 || /* version */
+ /* copy-data extension */
+ (r = sshbuf_put_cstring(msg, "copy-data")) != 0 ||
(r = sshbuf_put_cstring(msg, "1")) != 0) /* version */
fatal("%s: buffer error: %s", __func__, ssh_err(r));
send_msg(msg);
@@ -1369,6 +1374,88 @@ process_extended_fsync(u_int32_t id)
send_status(id, status);
}
+static void
+process_extended_copy_data(u_int32_t id)
+{
+ u_char buf[64*1024];
+ int read_handle, read_fd, write_handle, write_fd;
+ u_int64_t len, read_off, read_len, write_off;
+ int r, ret, copy_until_eof, status = SSH2_FX_OP_UNSUPPORTED;
+
+ if ((r = get_handle(iqueue, &read_handle)) != 0 ||
+ (r = sshbuf_get_u64(iqueue, &read_off)) != 0 ||
+ (r = sshbuf_get_u64(iqueue, &read_len)) != 0 ||
+ (r = get_handle(iqueue, &write_handle)) != 0 ||
+ (r = sshbuf_get_u64(iqueue, &write_off)) != 0)
+ fatal("%s: buffer error: %s", __func__, ssh_err(r));
+
+ debug("request %u: copy-data from \"%s\" (handle %d) off %llu
len %llu "
+ "to \"%s\" (handle %d) off %llu",
+ id, handle_to_name(read_handle), read_handle,
+ (unsigned long long)read_off, (unsigned long long)read_len,
+ handle_to_name(write_handle), write_handle,
+ (unsigned long long)write_off);
+
+ /* For read length of 0, we read until EOF. */
+ if (read_len == 0) {
+ read_len = (u_int64_t)-1 - read_off;
+ copy_until_eof = 1;
+ } else
+ copy_until_eof = 0;
+
+ read_fd = handle_to_fd(read_handle);
+ write_fd = handle_to_fd(write_handle);
+
+ /* Disallow reading & writing to the same handle */
+ if (read_handle == write_handle || read_fd < 0 || write_fd < 0) {
+ status = SSH2_FX_FAILURE;
+ } else {
+ if (lseek(read_fd, read_off, SEEK_SET) < 0) {
+ status = errno_to_portable(errno);
+ error("process_extended_copy_data: read_seek failed");
+ } else if (!(handle_to_flags(write_handle) & O_APPEND) &&
+ lseek(write_fd, write_off, SEEK_SET) < 0) {
+ status = errno_to_portable(errno);
+ error("process_extended_copy_data: write_seek failed");
+ } else {
+ /* Process the request in chunks. */
+ while (read_len || copy_until_eof) {
+ len = MINIMUM(sizeof(buf), read_len);
+ read_len -= len;
+
+ ret = read(read_fd, buf, len);
+ if (ret < 0) {
+ status = errno_to_portable(errno);
+ break;
+ } else if (ret == 0) {
+ status = copy_until_eof ? SSH2_FX_OK : SSH2_FX_EOF;
+ break;
+ }
+ len = ret;
+ handle_update_read(read_handle, len);
+
+ ret = write(write_fd, buf, len);
+ if (ret < 0) {
+ status = errno_to_portable(errno);
+ error("process_extended_copy_data: write failed");
+ break;
+ }
+ handle_update_write(write_handle, len);
+ if ((size_t)ret != len) {
+ debug2("nothing at all written");
+ status = SSH2_FX_FAILURE;
+ break;
+ }
+ }
+
+ if (read_len == 0)
+ status = SSH2_FX_OK;
+ }
+ }
+
+ send_status(id, status);
+}
+
static void
process_extended(u_int32_t id)
{
--
2.17.1
Mike Frysinger
2018-Nov-13 02:22 UTC
[PATCH 2/2] sftp-client: add a "cp" helper to copy files
Use the new copy-data extension for efficient remote file copies.
At the very least, this provides a quick method for testing.
---
sftp-client.c | 118 ++++++++++++++++++++++++++++++++++++++++++++++++++
sftp-client.h | 3 ++
sftp.1 | 7 ++-
sftp.c | 12 +++++
4 files changed, 139 insertions(+), 1 deletion(-)
diff --git a/sftp-client.c b/sftp-client.c
index 4986d6d8d291..cd2844a8585e 100644
--- a/sftp-client.c
+++ b/sftp-client.c
@@ -86,6 +86,7 @@ struct sftp_conn {
#define SFTP_EXT_FSTATVFS 0x00000004
#define SFTP_EXT_HARDLINK 0x00000008
#define SFTP_EXT_FSYNC 0x00000010
+#define SFTP_EXT_COPY_DATA 0x00000020
u_int exts;
u_int64_t limit_kbps;
struct bwlimit bwlimit_in, bwlimit_out;
@@ -463,6 +464,10 @@ do_init(int fd_in, int fd_out, u_int transfer_buflen, u_int
num_requests,
strcmp((char *)value, "1") == 0) {
ret->exts |= SFTP_EXT_FSYNC;
known = 1;
+ } else if (strcmp(name, "copy-data") == 0 &&
+ strcmp((char *)value, "1") == 0) {
+ ret->exts |= SFTP_EXT_COPY_DATA;
+ known = 1;
}
if (known) {
debug2("Server supports extension \"%s\" revision %s",
@@ -879,6 +884,119 @@ do_realpath(struct sftp_conn *conn, const char *path)
return(filename);
}
+int
+do_copy(struct sftp_conn *conn, const char *oldpath, const char *newpath)
+{
+ Attrib junk, *a;
+ struct sshbuf *msg;
+ u_char *old_handle, *new_handle;
+ u_int mode, status, id;
+ size_t old_handle_len, new_handle_len;
+ int r;
+
+ /* Silently return if the extension is not supported */
+ if ((conn->exts & SFTP_EXT_COPY_DATA) == 0) {
+ error("Server does not support copy-data extension");
+ return -1;
+ }
+
+ /* Make sure the file exists, and we can copy its perms */
+ if ((a = do_stat(conn, oldpath, 0)) == NULL)
+ return -1;
+
+ /* Do not preserve set[ug]id here, as we do not preserve ownership */
+ if (a->flags & SSH2_FILEXFER_ATTR_PERMISSIONS) {
+ mode = a->perm & 0777;
+
+ if (!S_ISREG(a->perm)) {
+ error("Cannot copy non-regular file: %s", oldpath);
+ return -1;
+ }
+ } else {
+ mode = 0666;
+ }
+
+ /* Set up the new perms for the new file */
+ attrib_clear(a);
+ a->perm = mode;
+ a->flags |= SSH2_FILEXFER_ATTR_PERMISSIONS;
+
+ if ((msg = sshbuf_new()) == NULL)
+ fatal("%s: sshbuf_new failed", __func__);
+
+ attrib_clear(&junk); /* Send empty attributes */
+
+ /* Open the old file for reading */
+ id = conn->msg_id++;
+ if ((r = sshbuf_put_u8(msg, SSH2_FXP_OPEN)) != 0 ||
+ (r = sshbuf_put_u32(msg, id)) != 0 ||
+ (r = sshbuf_put_cstring(msg, oldpath)) != 0 ||
+ (r = sshbuf_put_u32(msg, SSH2_FXF_READ)) != 0 ||
+ (r = encode_attrib(msg, &junk)) != 0)
+ fatal("%s: buffer error: %s", __func__, ssh_err(r));
+ send_msg(conn, msg);
+ debug3("Sent message SSH2_FXP_OPEN I:%u P:%s", id, oldpath);
+
+ sshbuf_reset(msg);
+
+ old_handle = get_handle(conn, id, &old_handle_len,
+ "remote open(\"%s\")", oldpath);
+ if (old_handle == NULL) {
+ sshbuf_free(msg);
+ return -1;
+ }
+
+ /* Open the new file for writing */
+ id = conn->msg_id++;
+ if ((r = sshbuf_put_u8(msg, SSH2_FXP_OPEN)) != 0 ||
+ (r = sshbuf_put_u32(msg, id)) != 0 ||
+ (r = sshbuf_put_cstring(msg, newpath)) != 0 ||
+ (r = sshbuf_put_u32(msg, SSH2_FXF_WRITE|SSH2_FXF_CREAT|
+ SSH2_FXF_TRUNC)) != 0 ||
+ (r = encode_attrib(msg, a)) != 0)
+ fatal("%s: buffer error: %s", __func__, ssh_err(r));
+ send_msg(conn, msg);
+ debug3("Sent message SSH2_FXP_OPEN I:%u P:%s", id, newpath);
+
+ sshbuf_reset(msg);
+
+ new_handle = get_handle(conn, id, &new_handle_len,
+ "remote open(\"%s\")", newpath);
+ if (new_handle == NULL) {
+ sshbuf_free(msg);
+ return -1;
+ }
+
+ /* Copy the file data */
+ id = conn->msg_id++;
+ if ((r = sshbuf_put_u8(msg, SSH2_FXP_EXTENDED)) != 0 ||
+ (r = sshbuf_put_u32(msg, id)) != 0 ||
+ (r = sshbuf_put_cstring(msg, "copy-data")) != 0 ||
+ (r = sshbuf_put_string(msg, old_handle, old_handle_len)) != 0 ||
+ (r = sshbuf_put_u64(msg, 0)) != 0 ||
+ (r = sshbuf_put_u64(msg, 0)) != 0 ||
+ (r = sshbuf_put_string(msg, new_handle, new_handle_len)) != 0 ||
+ (r = sshbuf_put_u64(msg, 0)) != 0)
+ fatal("%s: buffer error: %s", __func__, ssh_err(r));
+ send_msg(conn, msg);
+ debug3("Sent message copy-data \"%s\" 0 0 ->
\"%s\" 0",
+ oldpath, newpath);
+
+ status = get_status(conn, id);
+ if (status != SSH2_FX_OK)
+ error("Couldn't copy file \"%s\" to \"%s\":
%s", oldpath,
+ newpath, fx2txt(status));
+
+ /* Clean up everything */
+ sshbuf_free(msg);
+ do_close(conn, old_handle, old_handle_len);
+ do_close(conn, new_handle, new_handle_len);
+ free(old_handle);
+ free(new_handle);
+
+ return status == SSH2_FX_OK ? 0 : -1;
+}
+
int
do_rename(struct sftp_conn *conn, const char *oldpath, const char *newpath,
int force_legacy)
diff --git a/sftp-client.h b/sftp-client.h
index 14a3b8182c49..13b638ef006f 100644
--- a/sftp-client.h
+++ b/sftp-client.h
@@ -100,6 +100,9 @@ int do_statvfs(struct sftp_conn *, const char *, struct
sftp_statvfs *, int);
/* Rename 'oldpath' to 'newpath' */
int do_rename(struct sftp_conn *, const char *, const char *, int
force_legacy);
+/* Copy 'oldpath' to 'newpath' */
+int do_copy(struct sftp_conn *, const char *, const char *);
+
/* Link 'oldpath' to 'newpath' */
int do_hardlink(struct sftp_conn *, const char *, const char *);
diff --git a/sftp.1 b/sftp.1
index 0fd54cae090e..f2eae7f32790 100644
--- a/sftp.1
+++ b/sftp.1
@@ -137,7 +137,7 @@ will abort if any of the following
commands fail:
.Ic get , put , reget , reput, rename , ln ,
.Ic rm , mkdir , chdir , ls ,
-.Ic lchdir , chmod , chown ,
+.Ic lchdir , copy , chmod , chown ,
.Ic chgrp , lpwd , df , symlink ,
and
.Ic lmkdir .
@@ -340,6 +340,11 @@ may contain
characters and may match multiple files.
.Ar own
must be a numeric UID.
+.It Ic copy Ar oldpath Ar newpath
+Copy remote file from
+.Ar oldpath
+to
+.Ar newpath .
.It Xo Ic df
.Op Fl hi
.Op Ar path
diff --git a/sftp.c b/sftp.c
index 7db86c2d3cf0..3288279172a9 100644
--- a/sftp.c
+++ b/sftp.c
@@ -142,6 +142,7 @@ enum sftp_command {
I_CHGRP,
I_CHMOD,
I_CHOWN,
+ I_COPY,
I_DF,
I_GET,
I_HELP,
@@ -185,6 +186,7 @@ static const struct CMD cmds[] = {
{ "chgrp", I_CHGRP, REMOTE },
{ "chmod", I_CHMOD, REMOTE },
{ "chown", I_CHOWN, REMOTE },
+ { "cp", I_COPY, REMOTE },
{ "df", I_DF, REMOTE },
{ "dir", I_LS, REMOTE },
{ "exit", I_QUIT, NOARGS },
@@ -281,6 +283,7 @@ help(void)
"chgrp grp path Change group of file
'path' to 'grp'\n"
"chmod mode path Change permissions of file
'path' to 'mode'\n"
"chown own path Change owner of file
'path' to 'own'\n"
+ "cp oldpath newpath Copy remote file\n"
"df [-hi] [path] Display statistics for current
directory or\n"
" filesystem containing
'path'\n"
"exit Quit sftp\n"
@@ -1373,6 +1376,10 @@ parse_args(const char **cpp, int *ignore_errors, int
*aflag,
if ((optidx = parse_link_flags(cmd, argv, argc, sflag)) == -1)
return -1;
goto parse_two_paths;
+ case I_COPY:
+ if ((optidx = parse_no_flags(cmd, argv, argc)) == -1)
+ return -1;
+ goto parse_two_paths;
case I_RENAME:
if ((optidx = parse_rename_flags(cmd, argv, argc, lflag)) == -1)
return -1;
@@ -1535,6 +1542,11 @@ parse_dispatch_command(struct sftp_conn *conn, const char
*cmd, char **pwd,
err = process_put(conn, path1, path2, *pwd, pflag,
rflag, aflag, fflag);
break;
+ case I_COPY:
+ path1 = make_absolute(path1, *pwd);
+ path2 = make_absolute(path2, *pwd);
+ err = do_copy(conn, path1, path2);
+ break;
case I_RENAME:
path1 = make_absolute(path1, *pwd);
path2 = make_absolute(path2, *pwd);
--
2.17.1