[RemoteUtil] Add support for sftp (#64)

In a future CL, we will switch from scp to sftp. This CL adds support
for calling sftp from RemoteUtil.

In order to maintain backwards compatibility where people still set
--scp-command or CDC_SCP_COMMAND instead of the sftp versions, this CL
also adds the helper method RemoteUtil::ScpToSftpCommand, which
attempts to convert an scp command to an sftp command. This is usually
possible since the args are almost the same. For instance, if the scp
command is
  C:\path\to\scp.exe -P 1234 -i <key_file> -oUserKnownHostsFile=known_hosts
then the corresponding sftp command is most likely
  C:\path\to\sftp.exe -P 1234 -i <key_file> -oUserKnownHostsFile=known_hosts
This works for instance for OpenSSH.

common/remote_util.cc

@@ -20,6 +20,8 @@
 #include "absl/strings/str_cat.h"
 #include "absl/strings/str_format.h"
 #include "common/path.h"
+#include "common/status_macros.h"
+#include "common/util.h"
 
 namespace cdc_ft {
 namespace {
@@ -47,10 +49,34 @@ void RemoteUtil::SetScpCommand(std::string scp_command) {
   scp_command_ = std::move(scp_command);
 }
 
+void RemoteUtil::SetSftpCommand(std::string sftp_command) {
+  sftp_command_ = std::move(sftp_command);
+}
+
 void RemoteUtil::SetSshCommand(std::string ssh_command) {
   ssh_command_ = std::move(ssh_command);
 }
 
+// static
+std::string RemoteUtil::ScpToSftpCommand(std::string scp_command) {
+  // "scp", "SCP", "winscp.exe", "C:\path\to\scp", "/scppath/scp --foo" etc.
+  std::string lower_scp_command = scp_command;
+  std::transform(lower_scp_command.begin(), lower_scp_command.end(),
+                 lower_scp_command.begin(), ::tolower);
+  size_t pos = 0;
+  while ((pos = lower_scp_command.find("scp", pos)) != std::string::npos) {
+    // This may access the string at scp_command.size(), but that's well defined
+    // in C++11 and returns 0.
+    const char next_ch = lower_scp_command[pos + 3];
+    if ((next_ch == 0 || next_ch == '.' || next_ch == ' ')) {
+      return scp_command.replace(pos, 3, "sftp");
+    }
+    ++pos;
+  }
+
+  return std::string();
+}
+
 absl::Status RemoteUtil::Scp(std::vector<std::string> source_filepaths,
                              const std::string& dest, bool compress) {
   std::string source_args;
@@ -77,6 +103,33 @@ absl::Status RemoteUtil::Scp(std::vector<std::string> source_filepaths,
   return process_factory_->Run(start_info);
 }
 
+absl::Status RemoteUtil::Sftp(const std::string& commands,
+                              const std::string& initial_local_dir,
+                              bool compress) {
+  // sftp doesn't take |commands| as argument, so write it to a temp file.
+  std::string cmd_path =
+      path::Join(path::GetTempDir(), "__sftp_cmd__" + Util::GenerateUniqueId());
+  RETURN_IF_ERROR(path::WriteFile(cmd_path, commands),
+                  "Failed to write sftp commands to '%s'", cmd_path);
+
+  // -p preserves timestamps. This enables timestamp-based up-to-date checks.
+  ProcessStartInfo start_info;
+  start_info.flags = ProcessFlags::kNoWindow;
+  start_info.command = absl::StrFormat(
+      "%s %s %s -p -b %s %s", sftp_command_,
+      quiet_ || verbosity_ < 2 ? "-q" : "", compress ? "-C" : "",
+      QuoteForWindows(cmd_path), QuoteForWindows(user_host_));
+  start_info.name = "sftp";
+  start_info.startup_dir = initial_local_dir;
+  start_info.forward_output_to_log = forward_output_to_log_;
+
+  RETURN_IF_ERROR(process_factory_->Run(start_info));
+
+  // Note: Keep |cmd_path| in case of an error for debugging purposes.
+  path::RemoveFile(cmd_path).IgnoreError();
+  return absl::OkStatus();
+}
+
 absl::Status RemoteUtil::Chmod(const std::string& mode,
                                const std::string& remote_path, bool quiet) {
   std::string remote_command =

common/remote_util.h

@@ -39,20 +39,53 @@ class RemoteUtil {
              ProcessFactory* process_factory, bool forward_output_to_log);
 
   // Sets the SCP command binary path and additional arguments, e.g.
-  //   C:\path\to\scp.exe -p 1234 -i <key_file> -oUserKnownHostsFile=known_hosts
+  //   C:\path\to\scp.exe -P 1234 -i <key_file> -oUserKnownHostsFile=known_hosts
-  // By default, searches scp.exe on the path environment variables.
+  // By default, searches scp on the path environment variables.
   void SetScpCommand(std::string scp_command);
 
+  // Sets the SFTP command binary path and additional arguments, e.g.
+  //   C:\path\to\sftp.exe -P 1234 -i <key_file>
+  //   -oUserKnownHostsFile=known_hosts
+  // By default, searches sftp on the path environment variables.
+  void SetSftpCommand(std::string sftp_command);
+
   // Sets the SSH command binary path and additional arguments, e.g.
-  //   C:\path\to\ssh.exe -P 1234 -i <key_file> -oUserKnownHostsFile=known_hosts
+  //   C:\path\to\ssh.exe -p 1234 -i <key_file> -oUserKnownHostsFile=known_hosts
-  // By default, searches ssh.exe on the path environment variables.
+  // By default, searches ssh on the path environment variables.
   void SetSshCommand(std::string ssh_command);
 
+  // Converts an scp command into an sftp command by simply replacing the first
+  // occurrance of "scp.", "scp " or "scp\0" by sftp (case insensitive). This
+  // adds backwards compatibility after a switch from scp to sftp in case users
+  // still set CDC_SCP_COMMAND or --scp-command. Luckily, all relevant
+  // parameters of sftp and scp match.
+  // Returns an empty string if |scp_command| does not contain "scp".
+  // Returns bad results for tricky strings like "C:\scp.path\scp.exe".
+  static std::string ScpToSftpCommand(std::string scp_command);
+
   // Copies |source_filepaths| to the remote folder |dest| on the gamelet using
   // scp. If |compress| is true, compressed upload is used.
   absl::Status Scp(std::vector<std::string> source_filepaths,
                    const std::string& dest, bool compress);
 
+  // Creates an sftp connection to the remote instance and executes the
+  // newline-separated SFTP |commands|. See
+  //   https://man7.org/linux/man-pages/man1/sftp.1.html
+  // for a list of available commands.
+  // |initial_local_dir| sets the initial local directory in sftp. This is
+  // useful since some sftp clients don't work with standard Windows paths and
+  // require for instance /cygdrive paths.
+  // If |compress| is true, compressed upload is used.
+  // Example: Create nested directories and copying an executable file.
+  //   -mkdir a
+  //   cd a
+  //   -mkdir b
+  //   cd b
+  //   put foo_executable
+  //   chmod 755 foo_executable
+  absl::Status Sftp(const std::string& commands,
+                    const std::string& initial_local_dir, bool compress);
+
   // Calls 'chmod |mode| |remote_path|' on the gamelet.
   absl::Status Chmod(const std::string& mode, const std::string& remote_path,
                      bool quiet = false);
@@ -115,6 +148,7 @@ class RemoteUtil {
   const bool forward_output_to_log_;
 
   std::string scp_command_ = "scp";
+  std::string sftp_command_ = "sftp";
   std::string ssh_command_ = "ssh";
   std::string user_host_;
 };

common/remote_util_test.cc

@@ -127,5 +127,27 @@ TEST_F(RemoteUtilTest, QuoteForSsh) {
             "\"~user-name69/\\\"foo\\\"\"");  // Nice!
 }
 
+TEST_F(RemoteUtilTest, ScpToSftpCommand) {
+  EXPECT_EQ(RemoteUtil::ScpToSftpCommand(""), "");
+  EXPECT_EQ(RemoteUtil::ScpToSftpCommand("scp"), "sftp");
+  EXPECT_EQ(RemoteUtil::ScpToSftpCommand("scp.exe"), "sftp.exe");
+  EXPECT_EQ(RemoteUtil::ScpToSftpCommand("scp --arg"), "sftp --arg");
+  EXPECT_EQ(RemoteUtil::ScpToSftpCommand("ScP --aRg"), "sftp --aRg");
+  EXPECT_EQ(RemoteUtil::ScpToSftpCommand("winscp"), "winsftp");
+  EXPECT_EQ(RemoteUtil::ScpToSftpCommand("winscp.exe"), "winsftp.exe");
+  EXPECT_EQ(RemoteUtil::ScpToSftpCommand("winscp --arg"), "winsftp --arg");
+  EXPECT_EQ(RemoteUtil::ScpToSftpCommand("C:\\path\\to\\scp"),
+            "C:\\path\\to\\sftp");
+  EXPECT_EQ(RemoteUtil::ScpToSftpCommand("C:\\path\\to\\scp.exe"),
+            "C:\\path\\to\\sftp.exe");
+  EXPECT_EQ(RemoteUtil::ScpToSftpCommand("C:\\path\\to\\scp.exe --arg"),
+            "C:\\path\\to\\sftp.exe --arg");
+  EXPECT_EQ(RemoteUtil::ScpToSftpCommand("C:\\scp.exe --argwithscp"),
+            "C:\\sftp.exe --argwithscp");
+  EXPECT_EQ(RemoteUtil::ScpToSftpCommand("C:\\path_with_scp\\scp"),
+            "C:\\path_with_scp\\sftp");
+  EXPECT_EQ(RemoteUtil::ScpToSftpCommand("C:\\path\\to\\somethingelse"), "");
+}
+
 }  // namespace
 }  // namespace cdc_ft