From: Mike Frysinger <vapier at chromium.org> Add support to the sftp-server for the copy-data extension defined in [1]. This provides for efficient copying on the server side otherwise clients have to emulate it by downloading the file before uploading it to the new path. For large files or slow networks, this can be prohibitively slow. Now doing a file copy is limited only by the server I/O. For clients that provide virtual file systems on top of SFTP, this can provide significant performance gains. [1] https://tools.ietf.org/html/draft-ietf-secsh-filexfer-extensions-00#section-7 --- PROTOCOL | 39 +++++++++++++++++++++- sftp-server.c | 92 +++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 130 insertions(+), 1 deletion(-) diff --git a/PROTOCOL b/PROTOCOL index 3141cda6fa37..42c040d65a04 100644 --- a/PROTOCOL +++ b/PROTOCOL @@ -460,7 +460,7 @@ This request asks the server to call fsync(2) on an open file handle. string "fsync at openssh.com" string handle -One receiving this request, a server will call fsync(handle_fd) and will +On receiving this request, a server will call fsync(handle_fd) and will respond with a SSH_FXP_STATUS message. This extension is advertised in the SSH_FXP_VERSION hello with version @@ -544,6 +544,43 @@ Its reply is the same format as that of SSH2_FXP_REALPATH. This extension is advertised in the SSH_FXP_VERSION hello with version "1". +3.10. sftp: Extension request "copy-data" + +This request asks the server to copy data from one open file handle and +write it to a different open file handle. This avoids needing to transfer +the data across the network twice (a download followed by an upload). + + byte SSH_FXP_EXTENDED + uint32 id + string "copy-data" + string read-from-handle + uint64 read-from-offset + uint64 read-data-length + string write-to-handle + uint64 write-to-offset + +The server will copy read-data-length bytes starting from +read-from-offset from the read-from-handle and write them to +write-to-handle starting from write-to-offset, and then respond with a +SSH_FXP_STATUS message. + +It's equivalent to issuing a series of SSH_FXP_READ requests on +read-from-handle and a series of requests of SSH_FXP_WRITE on +write-to-handle. + +If read-from-handle and write-to-handle are the same, the server will +fail the request and respond with a SSH_FX_INVALID_PARAMETER message. + +If read-data-length is 0, then the server will read data from the +read-from-handle until EOF is reached. + +This extension is advertised in the SSH_FXP_VERSION hello with version +"1". + +This request is identical to the "copy-data" request documented in: + +https://tools.ietf.org/html/draft-ietf-secsh-filexfer-extensions-00#section-7 + 4. Miscellaneous changes 4.1 Public key format diff --git a/sftp-server.c b/sftp-server.c index 18d194911257..e15874d0006c 100644 --- a/sftp-server.c +++ b/sftp-server.c @@ -41,6 +41,7 @@ #include <unistd.h> #include <stdarg.h> +#include "atomicio.h" #include "xmalloc.h" #include "sshbuf.h" #include "ssherr.h" @@ -116,6 +117,7 @@ static void process_extended_fsync(u_int32_t id); static void process_extended_lsetstat(u_int32_t id); static void process_extended_limits(u_int32_t id); static void process_extended_expand(u_int32_t id); +static void process_extended_copy_data(u_int32_t id); static void process_extended(u_int32_t id); struct sftp_handler { @@ -161,6 +163,7 @@ static const struct sftp_handler extended_handlers[] = { { "limits", "limits at openssh.com", 0, process_extended_limits, 0 }, { "expand-path", "expand-path at openssh.com", 0, process_extended_expand, 0 }, + { "copy-data", "copy-data", 0, process_extended_copy_data, 1 }, { NULL, NULL, 0, NULL, 0 } }; @@ -710,6 +713,7 @@ process_init(void) compose_extension(msg, "lsetstat at openssh.com", "1"); compose_extension(msg, "limits at openssh.com", "1"); compose_extension(msg, "expand-path at openssh.com", "1"); + compose_extension(msg, "copy-data", "1"); send_msg(msg); sshbuf_free(msg); @@ -1580,6 +1584,94 @@ process_extended_expand(u_int32_t id) free(path); } +static void +process_extended_copy_data(u_int32_t id) +{ + u_char buf[64*1024]; + int read_handle, read_fd, write_handle, write_fd; + u_int64_t len, read_off, read_len, write_off; + int r, copy_until_eof, status = SSH2_FX_OP_UNSUPPORTED; + size_t ret; + + if ((r = get_handle(iqueue, &read_handle)) != 0 || + (r = sshbuf_get_u64(iqueue, &read_off)) != 0 || + (r = sshbuf_get_u64(iqueue, &read_len)) != 0 || + (r = get_handle(iqueue, &write_handle)) != 0 || + (r = sshbuf_get_u64(iqueue, &write_off)) != 0) + fatal("%s: buffer error: %s", __func__, ssh_err(r)); + + debug("request %u: copy-data from \"%s\" (handle %d) off %llu len %llu " + "to \"%s\" (handle %d) off %llu", + id, handle_to_name(read_handle), read_handle, + (unsigned long long)read_off, (unsigned long long)read_len, + handle_to_name(write_handle), write_handle, + (unsigned long long)write_off); + + /* For read length of 0, we read until EOF. */ + if (read_len == 0) { + read_len = (u_int64_t)-1 - read_off; + copy_until_eof = 1; + } else + copy_until_eof = 0; + + read_fd = handle_to_fd(read_handle); + write_fd = handle_to_fd(write_handle); + + /* Disallow reading & writing to the same handle or same path or dirs */ + if (read_handle == write_handle || read_fd < 0 || write_fd < 0 || + !strcmp(handle_to_name(read_handle), handle_to_name(write_handle))) { + status = SSH2_FX_FAILURE; + goto out; + } + + if (lseek(read_fd, read_off, SEEK_SET) < 0) { + status = errno_to_portable(errno); + error("%s: read_seek failed", __func__); + goto out; + } + + if (!(handle_to_flags(write_handle) & O_APPEND) && + lseek(write_fd, write_off, SEEK_SET) < 0) { + status = errno_to_portable(errno); + error("%s: write_seek failed", __func__); + goto out; + } + + /* Process the request in chunks. */ + while (read_len > 0 || copy_until_eof) { + len = MINIMUM(sizeof(buf), read_len); + read_len -= len; + + ret = atomicio(read, read_fd, buf, len); + if (ret == 0 && errno == EPIPE) { + status = copy_until_eof ? SSH2_FX_OK : SSH2_FX_EOF; + break; + } else if (ret == 0) { + status = errno_to_portable(errno); + error("%s: read failed: %s", __func__, strerror(errno)); + break; + } + len = ret; + handle_update_read(read_handle, len); + + ret = atomicio(vwrite, write_fd, buf, len); + if (ret != len) { + status = errno_to_portable(errno); + error("%s: write failed: %llu != %llu: %s", __func__, + (unsigned long long)ret, (unsigned long long)len, + strerror(errno)); + break; + } + handle_update_write(write_handle, len); + } + + if (read_len == 0) + status = SSH2_FX_OK; + + out: + send_status(id, status); +} + static void process_extended(u_int32_t id) { -- 2.33.0
Mike Frysinger
2021-Sep-25 02:53 UTC
[PATCH 2/2] sftp-client: add a "cp" helper to copy files
Use the new copy-data extension for efficient remote file copies. At the very least, this provides a quick method for testing. --- sftp-client.c | 120 ++++++++++++++++++++++++++++++++++++++++++++++++++ sftp-client.h | 3 ++ sftp.1 | 14 +++++- sftp.c | 14 ++++++ 4 files changed, 150 insertions(+), 1 deletion(-) diff --git a/sftp-client.c b/sftp-client.c index 9de9afa20f68..dc55ee7a4594 100644 --- a/sftp-client.c +++ b/sftp-client.c @@ -103,6 +103,7 @@ struct sftp_conn { #define SFTP_EXT_LSETSTAT 0x00000020 #define SFTP_EXT_LIMITS 0x00000040 #define SFTP_EXT_PATH_EXPAND 0x00000080 +#define SFTP_EXT_COPY_DATA 0x00000100 u_int exts; u_int64_t limit_kbps; struct bwlimit bwlimit_in, bwlimit_out; @@ -534,6 +535,10 @@ do_init(int fd_in, int fd_out, u_int transfer_buflen, u_int num_requests, strcmp((char *)value, "1") == 0) { ret->exts |= SFTP_EXT_PATH_EXPAND; known = 1; + } else if (strcmp(name, "copy-data") == 0 && + strcmp((char *)value, "1") == 0) { + ret->exts |= SFTP_EXT_COPY_DATA; + known = 1; } if (known) { debug2("Server supports extension \"%s\" revision %s", @@ -1060,6 +1065,121 @@ do_expand_path(struct sftp_conn *conn, const char *path) return do_realpath_expand(conn, path, 1); } +int +do_copy(struct sftp_conn *conn, const char *oldpath, const char *newpath) +{ + Attrib junk, *a; + struct sshbuf *msg; + u_char *old_handle, *new_handle; + u_int mode, status, id; + size_t old_handle_len, new_handle_len; + int r; + + /* Return if the extension is not supported */ + if ((conn->exts & SFTP_EXT_COPY_DATA) == 0) { + error("Server does not support copy-data extension"); + return -1; + } + + /* Make sure the file exists, and we can copy its perms */ + if ((a = do_stat(conn, oldpath, 0)) == NULL) + return -1; + + /* Do not preserve set[ug]id here, as we do not preserve ownership */ + if (a->flags & SSH2_FILEXFER_ATTR_PERMISSIONS) { + mode = a->perm & 0777; + + if (!S_ISREG(a->perm)) { + error("Cannot copy non-regular file: %s", oldpath); + return -1; + } + } else { + /* NB: The user's umask will apply to this */ + mode = 0666; + } + + /* Set up the new perms for the new file */ + attrib_clear(a); + a->perm = mode; + a->flags |= SSH2_FILEXFER_ATTR_PERMISSIONS; + + if ((msg = sshbuf_new()) == NULL) + fatal("%s: sshbuf_new failed", __func__); + + attrib_clear(&junk); /* Send empty attributes */ + + /* Open the old file for reading */ + id = conn->msg_id++; + if ((r = sshbuf_put_u8(msg, SSH2_FXP_OPEN)) != 0 || + (r = sshbuf_put_u32(msg, id)) != 0 || + (r = sshbuf_put_cstring(msg, oldpath)) != 0 || + (r = sshbuf_put_u32(msg, SSH2_FXF_READ)) != 0 || + (r = encode_attrib(msg, &junk)) != 0) + fatal("%s: buffer error: %s", __func__, ssh_err(r)); + send_msg(conn, msg); + debug3("Sent message SSH2_FXP_OPEN I:%u P:%s", id, oldpath); + + sshbuf_reset(msg); + + old_handle = get_handle(conn, id, &old_handle_len, + "remote open(\"%s\")", oldpath); + if (old_handle == NULL) { + sshbuf_free(msg); + return -1; + } + + /* Open the new file for writing */ + id = conn->msg_id++; + if ((r = sshbuf_put_u8(msg, SSH2_FXP_OPEN)) != 0 || + (r = sshbuf_put_u32(msg, id)) != 0 || + (r = sshbuf_put_cstring(msg, newpath)) != 0 || + (r = sshbuf_put_u32(msg, SSH2_FXF_WRITE|SSH2_FXF_CREAT| + SSH2_FXF_TRUNC)) != 0 || + (r = encode_attrib(msg, a)) != 0) + fatal("%s: buffer error: %s", __func__, ssh_err(r)); + send_msg(conn, msg); + debug3("Sent message SSH2_FXP_OPEN I:%u P:%s", id, newpath); + + sshbuf_reset(msg); + + new_handle = get_handle(conn, id, &new_handle_len, + "remote open(\"%s\")", newpath); + if (new_handle == NULL) { + sshbuf_free(msg); + free(old_handle); + return -1; + } + + /* Copy the file data */ + id = conn->msg_id++; + if ((r = sshbuf_put_u8(msg, SSH2_FXP_EXTENDED)) != 0 || + (r = sshbuf_put_u32(msg, id)) != 0 || + (r = sshbuf_put_cstring(msg, "copy-data")) != 0 || + (r = sshbuf_put_string(msg, old_handle, old_handle_len)) != 0 || + (r = sshbuf_put_u64(msg, 0)) != 0 || + (r = sshbuf_put_u64(msg, 0)) != 0 || + (r = sshbuf_put_string(msg, new_handle, new_handle_len)) != 0 || + (r = sshbuf_put_u64(msg, 0)) != 0) + fatal("%s: buffer error: %s", __func__, ssh_err(r)); + send_msg(conn, msg); + debug3("Sent message copy-data \"%s\" 0 0 -> \"%s\" 0", + oldpath, newpath); + + status = get_status(conn, id); + if (status != SSH2_FX_OK) + error("Couldn't copy file \"%s\" to \"%s\": %s", oldpath, + newpath, fx2txt(status)); + + /* Clean up everything */ + sshbuf_free(msg); + do_close(conn, old_handle, old_handle_len); + do_close(conn, new_handle, new_handle_len); + free(old_handle); + free(new_handle); + + return status == SSH2_FX_OK ? 0 : -1; +} + int do_rename(struct sftp_conn *conn, const char *oldpath, const char *newpath, int force_legacy) diff --git a/sftp-client.h b/sftp-client.h index 7d0bd12ae5a3..f9911d35c949 100644 --- a/sftp-client.h +++ b/sftp-client.h @@ -125,6 +125,9 @@ int do_statvfs(struct sftp_conn *, const char *, struct sftp_statvfs *, int); /* Rename 'oldpath' to 'newpath' */ int do_rename(struct sftp_conn *, const char *, const char *, int); +/* Copy 'oldpath' to 'newpath' */ +int do_copy(struct sftp_conn *, const char *, const char *); + /* Link 'oldpath' to 'newpath' */ int do_hardlink(struct sftp_conn *, const char *, const char *); diff --git a/sftp.1 b/sftp.1 index 7eebeeacbf3f..72b61b14a4bc 100644 --- a/sftp.1 +++ b/sftp.1 @@ -144,7 +144,7 @@ will abort if any of the following commands fail: .Ic get , put , reget , reput , rename , ln , .Ic rm , mkdir , chdir , ls , -.Ic lchdir , chmod , chown , +.Ic lchdir , copy , cp , chmod , chown , .Ic chgrp , lpwd , df , symlink , and .Ic lmkdir . @@ -400,6 +400,18 @@ If the flag is specified, then symlinks will not be followed. Note that this is only supported by servers that implement the "lsetstat at openssh.com" extension. +.It Ic copy Ar oldpath Ar newpath +Copy remote file from +.Ar oldpath +to +.Ar newpath . +.Pp +Note that this is only supported by servers that implement the "copy-data" +extension. +.It Ic cp Ar oldpath Ar newpath +Alias to +.Ic copy +command. .It Xo Ic df .Op Fl hi .Op Ar path diff --git a/sftp.c b/sftp.c index 418f312f7bc6..04a1cf72d45f 100644 --- a/sftp.c +++ b/sftp.c @@ -138,6 +138,7 @@ enum sftp_command { I_CHGRP, I_CHMOD, I_CHOWN, + I_COPY, I_DF, I_GET, I_HELP, @@ -181,6 +182,8 @@ static const struct CMD cmds[] = { { "chgrp", I_CHGRP, REMOTE }, { "chmod", I_CHMOD, REMOTE }, { "chown", I_CHOWN, REMOTE }, + { "copy", I_COPY, REMOTE }, + { "cp", I_COPY, REMOTE }, { "df", I_DF, REMOTE }, { "dir", I_LS, REMOTE }, { "exit", I_QUIT, NOARGS }, @@ -287,6 +290,8 @@ help(void) "chgrp [-h] grp path Change group of file 'path' to 'grp'\n" "chmod [-h] mode path Change permissions of file 'path' to 'mode'\n" "chown [-h] own path Change owner of file 'path' to 'own'\n" + "copy oldpath newpath Copy remote file\n" + "cp oldpath newpath Copy remote file\n" "df [-hi] [path] Display statistics for current directory or\n" " filesystem containing 'path'\n" "exit Quit sftp\n" @@ -1370,6 +1375,10 @@ parse_args(const char **cpp, int *ignore_errors, int *disable_echo, int *aflag, if ((optidx = parse_link_flags(cmd, argv, argc, sflag)) == -1) return -1; goto parse_two_paths; + case I_COPY: + if ((optidx = parse_no_flags(cmd, argv, argc)) == -1) + return -1; + goto parse_two_paths; case I_RENAME: if ((optidx = parse_rename_flags(cmd, argv, argc, lflag)) == -1) return -1; @@ -1537,6 +1546,11 @@ parse_dispatch_command(struct sftp_conn *conn, const char *cmd, char **pwd, err = process_put(conn, path1, path2, *pwd, pflag, rflag, aflag, fflag); break; + case I_COPY: + path1 = make_absolute(path1, *pwd); + path2 = make_absolute(path2, *pwd); + err = do_copy(conn, path1, path2); + break; case I_RENAME: path1 = make_absolute(path1, *pwd); path2 = make_absolute(path2, *pwd); -- 2.33.0
From: Mike Frysinger <vapier at chromium.org> Add support to the sftp-server for the copy-data extension defined in [1]. This provides for efficient copying on the server side otherwise clients have to emulate it by downloading the file before uploading it to the new path. For large files or slow networks, this can be prohibitively slow. Now doing a file copy is limited only by the server I/O. For clients that provide virtual file systems on top of SFTP, this can provide significant performance gains. [1] https://tools.ietf.org/html/draft-ietf-secsh-filexfer-extensions-00#section-7 --- Rebased onto the latest tree. PROTOCOL | 39 +++++++++++++++++++++- sftp-server.c | 92 +++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 130 insertions(+), 1 deletion(-) diff --git a/PROTOCOL b/PROTOCOL index e6a7d60eef0b..36192341959b 100644 --- a/PROTOCOL +++ b/PROTOCOL @@ -492,7 +492,7 @@ This request asks the server to call fsync(2) on an open file handle. string "fsync at openssh.com" string handle -One receiving this request, a server will call fsync(handle_fd) and will +On receiving this request, a server will call fsync(handle_fd) and will respond with a SSH_FXP_STATUS message. This extension is advertised in the SSH_FXP_VERSION hello with version @@ -576,6 +576,43 @@ Its reply is the same format as that of SSH2_FXP_REALPATH. This extension is advertised in the SSH_FXP_VERSION hello with version "1". +4.10. sftp: Extension request "copy-data" + +This request asks the server to copy data from one open file handle and +write it to a different open file handle. This avoids needing to transfer +the data across the network twice (a download followed by an upload). + + byte SSH_FXP_EXTENDED + uint32 id + string "copy-data" + string read-from-handle + uint64 read-from-offset + uint64 read-data-length + string write-to-handle + uint64 write-to-offset + +The server will copy read-data-length bytes starting from +read-from-offset from the read-from-handle and write them to +write-to-handle starting from write-to-offset, and then respond with a +SSH_FXP_STATUS message. + +It's equivalent to issuing a series of SSH_FXP_READ requests on +read-from-handle and a series of requests of SSH_FXP_WRITE on +write-to-handle. + +If read-from-handle and write-to-handle are the same, the server will +fail the request and respond with a SSH_FX_INVALID_PARAMETER message. + +If read-data-length is 0, then the server will read data from the +read-from-handle until EOF is reached. + +This extension is advertised in the SSH_FXP_VERSION hello with version +"1". + +This request is identical to the "copy-data" request documented in: + +https://tools.ietf.org/html/draft-ietf-secsh-filexfer-extensions-00#section-7 + 5. Miscellaneous changes 5.1 Public key format diff --git a/sftp-server.c b/sftp-server.c index ec91e9c2d33f..6d228ec2a9d5 100644 --- a/sftp-server.c +++ b/sftp-server.c @@ -44,6 +44,7 @@ #include <unistd.h> #include <stdarg.h> +#include "atomicio.h" #include "xmalloc.h" #include "sshbuf.h" #include "ssherr.h" @@ -119,6 +120,7 @@ static void process_extended_fsync(u_int32_t id); static void process_extended_lsetstat(u_int32_t id); static void process_extended_limits(u_int32_t id); static void process_extended_expand(u_int32_t id); +static void process_extended_copy_data(u_int32_t id); static void process_extended(u_int32_t id); struct sftp_handler { @@ -164,6 +166,7 @@ static const struct sftp_handler extended_handlers[] = { { "limits", "limits at openssh.com", 0, process_extended_limits, 0 }, { "expand-path", "expand-path at openssh.com", 0, process_extended_expand, 0 }, + { "copy-data", "copy-data", 0, process_extended_copy_data, 1 }, { NULL, NULL, 0, NULL, 0 } }; @@ -713,6 +716,7 @@ process_init(void) compose_extension(msg, "lsetstat at openssh.com", "1"); compose_extension(msg, "limits at openssh.com", "1"); compose_extension(msg, "expand-path at openssh.com", "1"); + compose_extension(msg, "copy-data", "1"); send_msg(msg); sshbuf_free(msg); @@ -1584,6 +1588,94 @@ process_extended_expand(u_int32_t id) free(path); } +static void +process_extended_copy_data(u_int32_t id) +{ + u_char buf[64*1024]; + int read_handle, read_fd, write_handle, write_fd; + u_int64_t len, read_off, read_len, write_off; + int r, copy_until_eof, status = SSH2_FX_OP_UNSUPPORTED; + size_t ret; + + if ((r = get_handle(iqueue, &read_handle)) != 0 || + (r = sshbuf_get_u64(iqueue, &read_off)) != 0 || + (r = sshbuf_get_u64(iqueue, &read_len)) != 0 || + (r = get_handle(iqueue, &write_handle)) != 0 || + (r = sshbuf_get_u64(iqueue, &write_off)) != 0) + fatal("%s: buffer error: %s", __func__, ssh_err(r)); + + debug("request %u: copy-data from \"%s\" (handle %d) off %llu len %llu " + "to \"%s\" (handle %d) off %llu", + id, handle_to_name(read_handle), read_handle, + (unsigned long long)read_off, (unsigned long long)read_len, + handle_to_name(write_handle), write_handle, + (unsigned long long)write_off); + + /* For read length of 0, we read until EOF. */ + if (read_len == 0) { + read_len = (u_int64_t)-1 - read_off; + copy_until_eof = 1; + } else + copy_until_eof = 0; + + read_fd = handle_to_fd(read_handle); + write_fd = handle_to_fd(write_handle); + + /* Disallow reading & writing to the same handle or same path or dirs */ + if (read_handle == write_handle || read_fd < 0 || write_fd < 0 || + !strcmp(handle_to_name(read_handle), handle_to_name(write_handle))) { + status = SSH2_FX_FAILURE; + goto out; + } + + if (lseek(read_fd, read_off, SEEK_SET) < 0) { + status = errno_to_portable(errno); + error("%s: read_seek failed", __func__); + goto out; + } + + if (!(handle_to_flags(write_handle) & O_APPEND) && + lseek(write_fd, write_off, SEEK_SET) < 0) { + status = errno_to_portable(errno); + error("%s: write_seek failed", __func__); + goto out; + } + + /* Process the request in chunks. */ + while (read_len > 0 || copy_until_eof) { + len = MINIMUM(sizeof(buf), read_len); + read_len -= len; + + ret = atomicio(read, read_fd, buf, len); + if (ret == 0 && errno == EPIPE) { + status = copy_until_eof ? SSH2_FX_OK : SSH2_FX_EOF; + break; + } else if (ret == 0) { + status = errno_to_portable(errno); + error("%s: read failed: %s", __func__, strerror(errno)); + break; + } + len = ret; + handle_update_read(read_handle, len); + + ret = atomicio(vwrite, write_fd, buf, len); + if (ret != len) { + status = errno_to_portable(errno); + error("%s: write failed: %llu != %llu: %s", __func__, + (unsigned long long)ret, (unsigned long long)len, + strerror(errno)); + break; + } + handle_update_write(write_handle, len); + } + + if (read_len == 0) + status = SSH2_FX_OK; + + out: + send_status(id, status); +} + static void process_extended(u_int32_t id) { -- 2.33.0