From af9038b4dd7a6a63d94f779d6b2375896a8fc1ad Mon Sep 17 00:00:00 2001 From: Lutz Justen Date: Tue, 17 Jan 2023 12:05:17 +0100 Subject: [PATCH] [RemoteUtil] Add support for sftp (#64) In a future CL, we will switch from scp to sftp. This CL adds support for calling sftp from RemoteUtil. In order to maintain backwards compatibility where people still set --scp-command or CDC_SCP_COMMAND instead of the sftp versions, this CL also adds the helper method RemoteUtil::ScpToSftpCommand, which attempts to convert an scp command to an sftp command. This is usually possible since the args are almost the same. For instance, if the scp command is C:\path\to\scp.exe -P 1234 -i -oUserKnownHostsFile=known_hosts then the corresponding sftp command is most likely C:\path\to\sftp.exe -P 1234 -i -oUserKnownHostsFile=known_hosts This works for instance for OpenSSH. --- common/remote_util.cc | 53 ++++++++++++++++++++++++++++++++++++++ common/remote_util.h | 42 +++++++++++++++++++++++++++--- common/remote_util_test.cc | 22 ++++++++++++++++ 3 files changed, 113 insertions(+), 4 deletions(-) diff --git a/common/remote_util.cc b/common/remote_util.cc index 576f616..bfeb706 100644 --- a/common/remote_util.cc +++ b/common/remote_util.cc @@ -20,6 +20,8 @@ #include "absl/strings/str_cat.h" #include "absl/strings/str_format.h" #include "common/path.h" +#include "common/status_macros.h" +#include "common/util.h" namespace cdc_ft { namespace { @@ -47,10 +49,34 @@ void RemoteUtil::SetScpCommand(std::string scp_command) { scp_command_ = std::move(scp_command); } +void RemoteUtil::SetSftpCommand(std::string sftp_command) { + sftp_command_ = std::move(sftp_command); +} + void RemoteUtil::SetSshCommand(std::string ssh_command) { ssh_command_ = std::move(ssh_command); } +// static +std::string RemoteUtil::ScpToSftpCommand(std::string scp_command) { + // "scp", "SCP", "winscp.exe", "C:\path\to\scp", "/scppath/scp --foo" etc. + std::string lower_scp_command = scp_command; + std::transform(lower_scp_command.begin(), lower_scp_command.end(), + lower_scp_command.begin(), ::tolower); + size_t pos = 0; + while ((pos = lower_scp_command.find("scp", pos)) != std::string::npos) { + // This may access the string at scp_command.size(), but that's well defined + // in C++11 and returns 0. + const char next_ch = lower_scp_command[pos + 3]; + if ((next_ch == 0 || next_ch == '.' || next_ch == ' ')) { + return scp_command.replace(pos, 3, "sftp"); + } + ++pos; + } + + return std::string(); +} + absl::Status RemoteUtil::Scp(std::vector source_filepaths, const std::string& dest, bool compress) { std::string source_args; @@ -77,6 +103,33 @@ absl::Status RemoteUtil::Scp(std::vector source_filepaths, return process_factory_->Run(start_info); } +absl::Status RemoteUtil::Sftp(const std::string& commands, + const std::string& initial_local_dir, + bool compress) { + // sftp doesn't take |commands| as argument, so write it to a temp file. + std::string cmd_path = + path::Join(path::GetTempDir(), "__sftp_cmd__" + Util::GenerateUniqueId()); + RETURN_IF_ERROR(path::WriteFile(cmd_path, commands), + "Failed to write sftp commands to '%s'", cmd_path); + + // -p preserves timestamps. This enables timestamp-based up-to-date checks. + ProcessStartInfo start_info; + start_info.flags = ProcessFlags::kNoWindow; + start_info.command = absl::StrFormat( + "%s %s %s -p -b %s %s", sftp_command_, + quiet_ || verbosity_ < 2 ? "-q" : "", compress ? "-C" : "", + QuoteForWindows(cmd_path), QuoteForWindows(user_host_)); + start_info.name = "sftp"; + start_info.startup_dir = initial_local_dir; + start_info.forward_output_to_log = forward_output_to_log_; + + RETURN_IF_ERROR(process_factory_->Run(start_info)); + + // Note: Keep |cmd_path| in case of an error for debugging purposes. + path::RemoveFile(cmd_path).IgnoreError(); + return absl::OkStatus(); +} + absl::Status RemoteUtil::Chmod(const std::string& mode, const std::string& remote_path, bool quiet) { std::string remote_command = diff --git a/common/remote_util.h b/common/remote_util.h index 8f7d714..cc4d0d0 100644 --- a/common/remote_util.h +++ b/common/remote_util.h @@ -39,20 +39,53 @@ class RemoteUtil { ProcessFactory* process_factory, bool forward_output_to_log); // Sets the SCP command binary path and additional arguments, e.g. - // C:\path\to\scp.exe -p 1234 -i -oUserKnownHostsFile=known_hosts - // By default, searches scp.exe on the path environment variables. + // C:\path\to\scp.exe -P 1234 -i -oUserKnownHostsFile=known_hosts + // By default, searches scp on the path environment variables. void SetScpCommand(std::string scp_command); + // Sets the SFTP command binary path and additional arguments, e.g. + // C:\path\to\sftp.exe -P 1234 -i + // -oUserKnownHostsFile=known_hosts + // By default, searches sftp on the path environment variables. + void SetSftpCommand(std::string sftp_command); + // Sets the SSH command binary path and additional arguments, e.g. - // C:\path\to\ssh.exe -P 1234 -i -oUserKnownHostsFile=known_hosts - // By default, searches ssh.exe on the path environment variables. + // C:\path\to\ssh.exe -p 1234 -i -oUserKnownHostsFile=known_hosts + // By default, searches ssh on the path environment variables. void SetSshCommand(std::string ssh_command); + // Converts an scp command into an sftp command by simply replacing the first + // occurrance of "scp.", "scp " or "scp\0" by sftp (case insensitive). This + // adds backwards compatibility after a switch from scp to sftp in case users + // still set CDC_SCP_COMMAND or --scp-command. Luckily, all relevant + // parameters of sftp and scp match. + // Returns an empty string if |scp_command| does not contain "scp". + // Returns bad results for tricky strings like "C:\scp.path\scp.exe". + static std::string ScpToSftpCommand(std::string scp_command); + // Copies |source_filepaths| to the remote folder |dest| on the gamelet using // scp. If |compress| is true, compressed upload is used. absl::Status Scp(std::vector source_filepaths, const std::string& dest, bool compress); + // Creates an sftp connection to the remote instance and executes the + // newline-separated SFTP |commands|. See + // https://man7.org/linux/man-pages/man1/sftp.1.html + // for a list of available commands. + // |initial_local_dir| sets the initial local directory in sftp. This is + // useful since some sftp clients don't work with standard Windows paths and + // require for instance /cygdrive paths. + // If |compress| is true, compressed upload is used. + // Example: Create nested directories and copying an executable file. + // -mkdir a + // cd a + // -mkdir b + // cd b + // put foo_executable + // chmod 755 foo_executable + absl::Status Sftp(const std::string& commands, + const std::string& initial_local_dir, bool compress); + // Calls 'chmod |mode| |remote_path|' on the gamelet. absl::Status Chmod(const std::string& mode, const std::string& remote_path, bool quiet = false); @@ -115,6 +148,7 @@ class RemoteUtil { const bool forward_output_to_log_; std::string scp_command_ = "scp"; + std::string sftp_command_ = "sftp"; std::string ssh_command_ = "ssh"; std::string user_host_; }; diff --git a/common/remote_util_test.cc b/common/remote_util_test.cc index 3b68f1d..3017e5b 100644 --- a/common/remote_util_test.cc +++ b/common/remote_util_test.cc @@ -127,5 +127,27 @@ TEST_F(RemoteUtilTest, QuoteForSsh) { "\"~user-name69/\\\"foo\\\"\""); // Nice! } +TEST_F(RemoteUtilTest, ScpToSftpCommand) { + EXPECT_EQ(RemoteUtil::ScpToSftpCommand(""), ""); + EXPECT_EQ(RemoteUtil::ScpToSftpCommand("scp"), "sftp"); + EXPECT_EQ(RemoteUtil::ScpToSftpCommand("scp.exe"), "sftp.exe"); + EXPECT_EQ(RemoteUtil::ScpToSftpCommand("scp --arg"), "sftp --arg"); + EXPECT_EQ(RemoteUtil::ScpToSftpCommand("ScP --aRg"), "sftp --aRg"); + EXPECT_EQ(RemoteUtil::ScpToSftpCommand("winscp"), "winsftp"); + EXPECT_EQ(RemoteUtil::ScpToSftpCommand("winscp.exe"), "winsftp.exe"); + EXPECT_EQ(RemoteUtil::ScpToSftpCommand("winscp --arg"), "winsftp --arg"); + EXPECT_EQ(RemoteUtil::ScpToSftpCommand("C:\\path\\to\\scp"), + "C:\\path\\to\\sftp"); + EXPECT_EQ(RemoteUtil::ScpToSftpCommand("C:\\path\\to\\scp.exe"), + "C:\\path\\to\\sftp.exe"); + EXPECT_EQ(RemoteUtil::ScpToSftpCommand("C:\\path\\to\\scp.exe --arg"), + "C:\\path\\to\\sftp.exe --arg"); + EXPECT_EQ(RemoteUtil::ScpToSftpCommand("C:\\scp.exe --argwithscp"), + "C:\\sftp.exe --argwithscp"); + EXPECT_EQ(RemoteUtil::ScpToSftpCommand("C:\\path_with_scp\\scp"), + "C:\\path_with_scp\\sftp"); + EXPECT_EQ(RemoteUtil::ScpToSftpCommand("C:\\path\\to\\somethingelse"), ""); +} + } // namespace } // namespace cdc_ft