github-mirrors/netris-cdc-file-transfer
1
0
Fork 0
mirror of https://github.com/nestriness/cdc-file-transfer.git synced 2026-05-01 17:03:07 +03:00
Code Issues Packages Projects Releases Wiki Activity

[cdc_rsync] [cdc_stream] Switch from scp to sftp (#66)

Browse Source 
Use sftp for deploying remote components instead of scp. sftp has the
advantage that it can also create directries, chmod files etc., so
that we can do everything in one call of sftp instead of mixing scp
and ssh calls.

The downside of sftp is that it can't switch to ~ resp. %userprofile%
for the remote side, and we have to assume that sftp starts in the
user's home dir. This is the default and works on my machines!

cdc_rsync and cdc_stream check the CDC_SFTP_COMMAND env var now and
accept --sftp-command flags. If they are not set, the corresponding
scp flag and env var is still used, with scp replaced by sftp. This is
most likely correct as sftp and scp usually reside in the same
directory and share largely identical parameters.
This commit is contained in:
Lutz Justen
2023-01-18 17:49:52 +01:00
committed by GitHub
parent a8b948b323
commit efca9855e7
19 changed files with 253 additions and 193 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
cdc_rsync/cdc_rsync_client.cc
View File

@@ -109,8 +109,8 @@ CdcRsyncClient::CdcRsyncClient(const Options& options,
if (!options_.ssh_command.empty()) {
remote_util_.SetSshCommand(options_.ssh_command);
}
if (!options_.scp_command.empty()) {
remote_util_.SetScpCommand(options_.scp_command);
if (!options_.sftp_command.empty()) {
remote_util_.SetSftpCommand(options_.sftp_command);
}
}
@@ -413,26 +413,10 @@ absl::Status CdcRsyncClient::DeployServer(const ServerArch& arch) {
}
printer_.Print(deploy_msg, true, Util::GetConsoleWidth());
// scp cdc_rsync_server to a temp location on the gamelet.
std::string remoteServerPath =
arch.RemoteToolsBinDir(ServerArch::UseCase::kScp) +
arch.CdcServerFilename();
std::string remoteServerTmpPath = remoteServerPath + Util::GenerateUniqueId();
std::string localServerPath = path::Join(exe_dir, arch.CdcServerFilename());
status = remote_util_.Scp({localServerPath}, remoteServerTmpPath,
/*compress=*/true);
if (!status.ok()) {
return WrapStatus(status, "Failed to copy cdc_rsync_server to instance");
}
// Replace cdc_rsync_server file by the temp file.
std::string replace_cmd =
arch.GetDeployReplaceCommand(remoteServerPath, remoteServerTmpPath);
status = remote_util_.Run(replace_cmd, "replace");
if (!status.ok()) {
return WrapStatus(status,
"Failed to replace old cdc_rsync_server by new one");
}
// sftp cdc_rsync_server to the target.
std::string commands = arch.GetDeploySftpCommands();
RETURN_IF_ERROR(remote_util_.Sftp(commands, exe_dir, /*compress=*/false),
"Failed to deploy cdc_rsync_server");
return absl::OkStatus();
}

6
cdc_rsync/cdc_rsync_client.h
View File

@@ -54,10 +54,14 @@ class CdcRsyncClient {
int forward_port_first = 44450;
int forward_port_last = 44459;
std::string ssh_command;
std::string scp_command;
std::string sftp_command;
std::string sources_dir; // Base dir for files loaded for --files-from.
PathFilter filter;
// Backwards compatibility for switching from scp to sftp.
// Used internally, do not use.
std::string deprecated_scp_command;
// Compression level 0 is invalid.
static constexpr int kMinCompressLevel = -5;
static constexpr int kMaxCompressLevel = 22;

42
cdc_rsync/params.cc
View File

@@ -21,6 +21,7 @@
#include "absl/strings/str_split.h"
#include "common/path.h"
#include "common/port_range_parser.h"
#include "common/remote_util.h"
#include "lib/zstd.h"
namespace cdc_ft {
@@ -75,9 +76,9 @@ Options:
--ssh-command <cmd> Path and arguments of ssh command to use, e.g.
"C:\path\to\ssh.exe -p 12345 -i id_rsa -oUserKnownHostsFile=known_hosts"
Can also be specified by the CDC_SSH_COMMAND environment variable.
--scp-command <cmd> Path and arguments of scp command to use, e.g.
"C:\path\to\scp.exe -P 12345 -i id_rsa -oUserKnownHostsFile=known_hosts"
Can also be specified by the CDC_SCP_COMMAND environment variable.
--sftp-command <cmd> Path and arguments of sftp command to use, e.g.
"C:\path\to\sftp.exe -P 12345 -i id_rsa -oUserKnownHostsFile=known_hosts"
Can also be specified by the CDC_SFTP_COMMAND environment variable.
--forward-port <port> TCP port or range used for SSH port forwarding (default: 44450-44459).
If a range is specified, searches for available ports (slower).
-h --help Help for cdc_rsync
@@ -85,12 +86,15 @@ Options:
constexpr char kSshCommandEnvVar[] = "CDC_SSH_COMMAND";
constexpr char kScpCommandEnvVar[] = "CDC_SCP_COMMAND";
constexpr char kSftpCommandEnvVar[] = "CDC_SFTP_COMMAND";
// Populates some parameters from environment variables.
void PopulateFromEnvVars(Parameters* parameters) {
path::GetEnv(kSshCommandEnvVar, &parameters->options.ssh_command)
.IgnoreError();
path::GetEnv(kScpCommandEnvVar, &parameters->options.scp_command)
path::GetEnv(kScpCommandEnvVar, &parameters->options.deprecated_scp_command)
.IgnoreError();
path::GetEnv(kSftpCommandEnvVar, &parameters->options.sftp_command)
.IgnoreError();
}
@@ -287,8 +291,15 @@ OptionResult HandleParameter(const std::string& key, const char* value,
}
if (key == "scp-command") {
// Backwards compatibility. Note that this flag is hidden from the help.
if (!ValidateValue(key, value)) return OptionResult::kError;
params->options.scp_command = value;
params->options.deprecated_scp_command = value;
return OptionResult::kConsumedKeyValue;
}
if (key == "sftp-command") {
if (!ValidateValue(key, value)) return OptionResult::kError;
params->options.sftp_command = value;
return OptionResult::kConsumedKeyValue;
}
@@ -491,6 +502,27 @@ bool Parse(int argc, const char* const* argv, Parameters* parameters) {
PopUserHost(&parameters->destination, &parameters->user_host);
// Backwards compabitility after switching to sftp. Convert scp to sftp
// command Note that this flag is hidden from the help.
if (parameters->options.sftp_command.empty() &&
!parameters->options.deprecated_scp_command.empty()) {
LOG_WARNING(
"The CDC_SCP_COMMAND environment variable and the --scp-command flag "
"are deprecated. Please set CDC_SFTP_COMMAND or --sftp-command "
"instead.");
parameters->options.sftp_command = RemoteUtil::ScpToSftpCommand(
parameters->options.deprecated_scp_command);
if (!parameters->options.sftp_command.empty()) {
LOG_WARNING("Converted scp command '%s' to sftp command '%s'.",
parameters->options.deprecated_scp_command,
parameters->options.sftp_command);
} else {
LOG_WARNING("Failed to convert scp command '%s' to sftp command.",
parameters->options.deprecated_scp_command);
}
}
if (!ValidateParameters(*parameters, help)) {
return false;
}

66
cdc_rsync/params_test.cc
View File

@@ -32,6 +32,10 @@ constexpr char kUserHostDst[] = "user@host:destination";
constexpr char kUserHost[] = "user@host";
constexpr char kDst[] = "destination";
constexpr char kSshCommandEnvVar[] = "CDC_SSH_COMMAND";
constexpr char kScpCommandEnvVar[] = "CDC_SCP_COMMAND";
constexpr char kSftpCommandEnvVar[] = "CDC_SFTP_COMMAND";
class TestLog : public Log {
public:
explicit TestLog() : Log(LogLevel::kInfo) {}
@@ -60,6 +64,11 @@ class ParamsTest : public ::testing::Test {
void TearDown() override {
std::cout.rdbuf(prev_stdout_);
std::cerr.rdbuf(prev_stderr_);
// Clear env. They seem to be sticky sometimes and leak into other tests.
path::SetEnv(kSshCommandEnvVar, "");
path::SetEnv(kScpCommandEnvVar, "");
path::SetEnv(kSftpCommandEnvVar, "");
}
protected:
@@ -152,24 +161,57 @@ TEST_F(ParamsTest, ParseFailsOnContimeoutEqualsNoValue) {
ExpectError(NeedsValueError("contimeout"));
}
TEST_F(ParamsTest, ParseSucceedsWithSshScpCommands) {
const char* argv[] = {"cdc_rsync.exe", kSrc,
kUserHostDst, "--ssh-command=sshcmd",
"--scp-command=scpcmd", NULL};
TEST_F(ParamsTest, ParseSucceedsWithSshSftpCommands) {
const char* argv[] = {
"cdc_rsync.exe", kSrc, kUserHostDst, "--ssh-command=sshcmd",
"--sftp-command=sftpcmd", NULL};
EXPECT_TRUE(Parse(static_cast<int>(std::size(argv)) - 1, argv, &parameters_));
EXPECT_EQ(parameters_.options.scp_command, "scpcmd");
EXPECT_EQ(parameters_.options.sftp_command, "sftpcmd");
EXPECT_EQ(parameters_.options.ssh_command, "sshcmd");
}
TEST_F(ParamsTest, ParseSucceedsWithSshScpCommandsByEnvVars) {
EXPECT_OK(path::SetEnv("CDC_SSH_COMMAND", "sshcmd"));
EXPECT_OK(path::SetEnv("CDC_SCP_COMMAND", "scpcmd"));
TEST_F(ParamsTest, ParseSucceedsWithSshSftpCommandsByEnvVars) {
EXPECT_OK(path::SetEnv(kSshCommandEnvVar, "sshcmd"));
EXPECT_OK(path::SetEnv(kSftpCommandEnvVar, "sftpcmd"));
const char* argv[] = {"cdc_rsync.exe", kSrc, kUserHostDst, NULL};
EXPECT_TRUE(Parse(static_cast<int>(std::size(argv)) - 1, argv, &parameters_));
EXPECT_EQ(parameters_.options.scp_command, "scpcmd");
EXPECT_EQ(parameters_.options.sftp_command, "sftpcmd");
EXPECT_EQ(parameters_.options.ssh_command, "sshcmd");
}
TEST_F(ParamsTest, ParseSucceedsWithScpCommandFallback) {
const char* argv[] = {"cdc_rsync.exe", kSrc, kUserHostDst,
"--scp-command=C:\\scp.exe foo", NULL};
EXPECT_TRUE(Parse(static_cast<int>(std::size(argv)) - 1, argv, &parameters_));
EXPECT_EQ(parameters_.options.sftp_command, "C:\\sftp.exe foo");
}
TEST_F(ParamsTest, ParseSucceedsWithScpCommandFallbackByEnvVar) {
EXPECT_OK(path::SetEnv(kScpCommandEnvVar, "C:\\scp.exe foo"));
const char* argv[] = {"cdc_rsync.exe", kSrc, kUserHostDst, NULL};
EXPECT_TRUE(Parse(static_cast<int>(std::size(argv)) - 1, argv, &parameters_));
EXPECT_EQ(parameters_.options.sftp_command, "C:\\sftp.exe foo");
}
TEST_F(ParamsTest, ParseSucceedsWithSftpOverwritingScp) {
const char* argv[] = {"cdc_rsync.exe",
kSrc,
kUserHostDst,
"--scp-command=C:\\scp.exe foo",
"--sftp-command=sftpcmd",
NULL};
EXPECT_TRUE(Parse(static_cast<int>(std::size(argv)) - 1, argv, &parameters_));
EXPECT_EQ(parameters_.options.sftp_command, "sftpcmd");
}
TEST_F(ParamsTest, ParseSucceedsWithSftpEnvVarOverwritingScp) {
EXPECT_OK(path::SetEnv(kSftpCommandEnvVar, "sftpcmd"));
const char* argv[] = {"cdc_rsync.exe", kSrc, kUserHostDst,
"--scp-command=C:\\scp.exe foo", NULL};
EXPECT_TRUE(Parse(static_cast<int>(std::size(argv)) - 1, argv, &parameters_));
EXPECT_EQ(parameters_.options.sftp_command, "sftpcmd");
}
TEST_F(ParamsTest, ParseSucceedsWithNoSshCommand) {
const char* argv[] = {"cdc_rsync.exe", kSrc, kUserHostDst,
"--ssh-command=", NULL};
@@ -178,12 +220,12 @@ TEST_F(ParamsTest, ParseSucceedsWithNoSshCommand) {
ExpectError(NeedsValueError("ssh-command"));
}
TEST_F(ParamsTest, ParseSucceedsWithNoScpCommand) {
const char* argv[] = {"cdc_rsync.exe", kSrc, kUserHostDst, "--scp-command",
TEST_F(ParamsTest, ParseSucceedsWithNoSftpCommand) {
const char* argv[] = {"cdc_rsync.exe", kSrc, kUserHostDst, "--sftp-command",
NULL};
EXPECT_FALSE(
Parse(static_cast<int>(std::size(argv)) - 1, argv, &parameters_));
ExpectError(NeedsValueError("scp-command"));
ExpectError(NeedsValueError("sftp-command"));
}
TEST_F(ParamsTest, ParseFailsOnNoUserHost) {

87
cdc_rsync/server_arch.cc
View File

@@ -18,8 +18,10 @@
#include "absl/strings/match.h"
#include "absl/strings/str_format.h"
#include "absl/strings/str_split.h"
#include "common/path.h"
#include "common/remote_util.h"
#include "common/util.h"
namespace cdc_ft {
@@ -72,19 +74,13 @@ std::string ServerArch::CdcServerFilename() const {
}
}
std::string ServerArch::RemoteToolsBinDir(UseCase use_case) const {
std::string ServerArch::RemoteToolsBinDir() const {
switch (type_) {
case Type::kWindows: {
// TODO(ljusten): Unfortunately, scp doesn't seem to support shell var
// expansion, so %AppData% can't be used. This relies on scp copying
// files relative to %UserProfile% and %AppData% mapping to
// "AppData\\Roaming" relative to that.
std::string app_data =
use_case == UseCase::kScp ? "AppData\\Roaming" : "$env:appdata";
return app_data + "\\cdc-file-transfer\\bin\\";
return "AppData\\Roaming\\cdc-file-transfer\\bin\\";
}
case Type::kLinux:
return "~/.cache/cdc-file-transfer/bin/";
return ".cache/cdc-file-transfer/bin/";
default:
assert(!kErrorArchTypeUnhandled);
return std::string();
@@ -93,11 +89,7 @@ std::string ServerArch::RemoteToolsBinDir(UseCase use_case) const {
std::string ServerArch::GetStartServerCommand(int exit_code_not_found,
const std::string& args) const {
std::string server_dir = RemoteToolsBinDir(UseCase::kSsh);
std::string server_path =
RemoteUtil::QuoteForSsh(server_dir + CdcServerFilename());
path::EnsureDoesNotEndWithPathSeparator(&server_dir);
server_dir = RemoteUtil::QuoteForSsh(server_dir);
std::string server_path = RemoteToolsBinDir() + CdcServerFilename();
switch (type_) {
case Type::kWindows:
@@ -106,53 +98,50 @@ std::string ServerArch::GetStartServerCommand(int exit_code_not_found,
// a minor issue and means we display "Deploying server..." instead of
// "Server not deployed. Deploying...";
return RemoteUtil::QuoteForWindows(
absl::StrFormat("Set-StrictMode -Version 2; "
absl::StrFormat("powershell -Command \" "
"Set-StrictMode -Version 2; "
"$ErrorActionPreference = 'Stop'; "
"$server_dir = %s; "
"$server_path = %s; "
"$u=New-Item $server_dir -ItemType Directory -Force; "
"if (-not (Test-Path -Path $server_path)) { "
"if (-not (Test-Path -Path '%s')) { "
" exit %i; "
"} "
"& $server_path %s",
server_dir, server_path, exit_code_not_found, args));
"%s %s "
"\"",
server_path, exit_code_not_found, server_path, args));
case Type::kLinux:
return absl::StrFormat(
"mkdir -p %s; if [ ! -f %s ]; then exit %i; fi; %s %s", server_dir,
server_path, exit_code_not_found, server_path, args);
return absl::StrFormat("if [ ! -f %s ]; then exit %i; fi; %s %s",
server_path, exit_code_not_found, server_path,
args);
default:
assert(!kErrorArchTypeUnhandled);
return std::string();
}
}
std::string ServerArch::GetDeployReplaceCommand(
const std::string& old_server_path,
const std::string& new_server_path) const {
const std::string old_path = RemoteUtil::QuoteForSsh(old_server_path);
const std::string new_path = RemoteUtil::QuoteForSsh(new_server_path);
std::string ServerArch::GetDeploySftpCommands() const {
std::string commands;
switch (type_) {
case Type::kWindows:
return RemoteUtil::QuoteForWindows(absl::StrFormat(
"Set-StrictMode -Version 2; "
"$ErrorActionPreference = 'Stop'; "
"$old_path = %s; "
"$new_path = %s; "
"if (Test-Path -Path $old_path) { "
" Get-Item -Path $old_path | "
" Set-ItemProperty -Name IsReadOnly -Value $false; "
"} "
"Move-Item -Path $new_path -Destination $old_path -Force",
old_path, new_path));
case Type::kLinux:
return absl::StrFormat(
"([ ! -f %s ] || chmod u+w %s) && chmod a+x %s && mv %s %s", old_path,
old_path, new_path, new_path, old_path);
default:
assert(!kErrorArchTypeUnhandled);
return std::string();
// Create the remote tools bin dir if it doesn't exist yet.
// This assumes that sftp's remote startup directory is the home directory.
const std::string server_dir = path::ToUnix(RemoteToolsBinDir());
std::vector<std::string> dir_parts =
absl::StrSplit(server_dir, '/', absl::SkipEmpty());
for (const std::string& dir : dir_parts) {
// Use -mkdir to ignore errors if the directory already exists.
commands += absl::StrFormat("-mkdir %s\ncd %s\n", dir, dir);
}
// Copy the server binary to a temp location. This assumes that sftp's local
// startup directory is cdc_rsync's exe dir.
const std::string server_file = CdcServerFilename();
const std::string server_temp_file = server_file + Util::GenerateUniqueId();
commands += absl::StrFormat("put %s %s\n", server_file, server_temp_file);
// Restore permissions in case they changed and propagate temp file.
commands += absl::StrFormat("-chmod 755 %s\n", server_file);
commands += absl::StrFormat("chmod 755 %s\n", server_temp_file);
commands += absl::StrFormat("rename %s %s\n", server_temp_file, server_file);
return commands;
}
} // namespace cdc_ft

26
cdc_rsync/server_arch.h
View File

@@ -29,8 +29,6 @@ class ServerArch {
kWindows = 1,
};
enum class UseCase { kSsh = 0, kScp = 1 };
// Detects the architecture type based on the destination path, e.g. path
// starting with C: indicate Windows.
static Type Detect(const std::string& destination);
@@ -42,16 +40,10 @@ class ServerArch {
std::string CdcServerFilename() const;
// Returns the arch-specific directory where cdc_rsync_server is deployed.
// On Windows, |use_case| determines what type of env variables to use:
// - kSsh uses $env:appdata and works for ssh commands.
// - kScp uses AppData\\Roaming and works for scp commands.
// On Linux, this flag is ignored.
std::string RemoteToolsBinDir(UseCase use_case) const;
std::string RemoteToolsBinDir() const;
// Returns an arch-specific SSH shell command that gets invoked in order to
// start cdc_rsync_server. The command
// - creates RemoteToolsBinDir() if it does not exist (so that the server can
// be deployed there subsequently; scp can't create directories),
// - returns |exit_code_not_found| if cdc_rsync_server does not exist (to
// prevent the confusing bash output message
// "bash: .../cdc_rsync_server: No such file or directory"), and
@@ -59,13 +51,15 @@ class ServerArch {
std::string GetStartServerCommand(int exit_code_not_found,
const std::string& args) const;
// Returns an arch-specific SSH shell command that gets invoked after
// cdc_rsync_server has been copied to a temp location. The command
// - makes the old cdc_rsync_server writable (if it exists),
// - makes the new cdc_rsync_server executable (Linux only) and
// - replaces the old cdc_rsync_server by the new one.
std::string GetDeployReplaceCommand(const std::string& old_server_path,
const std::string& new_server_path) const;
// Returns an arch-specific SFTP command sequence that deploys the server
// component on the target gets invoked after
// cdc_rsync_server has been copied to a temp location. The commands
// - create the cdc-file-transfer/bin folder if it doesn't exist yet,
// - make the old cdc_rsync_server writable if it exists,
// - copy cdc_rsync_server to a temp location,
// - make the new cdc_rsync_server executable (Linux only) and
// - replaces the existing cdc_rsync_server by the temp one.
std::string GetDeploySftpCommands() const;
private:
Type type_;

38
cdc_rsync/server_arch_test.cc
View File

@@ -60,47 +60,29 @@ TEST(ServerArchTest, CdcServerFilename) {
}
TEST(ServerArchTest, RemoteToolsBinDir) {
const std::string linux_ssh_dir =
ServerArch(kLinux).RemoteToolsBinDir(ServerArch::UseCase::kSsh);
EXPECT_TRUE(absl::StrContains(linux_ssh_dir, "/"));
const std::string linux_dir = ServerArch(kLinux).RemoteToolsBinDir();
EXPECT_TRUE(absl::StrContains(linux_dir, ".cache/"));
const std::string linux_scp_dir =
ServerArch(kLinux).RemoteToolsBinDir(ServerArch::UseCase::kScp);
EXPECT_EQ(linux_ssh_dir, linux_scp_dir);
const std::string win_ssh_dir =
ServerArch(kWindows).RemoteToolsBinDir(ServerArch::UseCase::kSsh);
EXPECT_TRUE(absl::StrContains(win_ssh_dir, "\\"));
EXPECT_TRUE(absl::StrContains(win_ssh_dir, "$env:appdata"));
std::string win_scp_dir =
ServerArch(kWindows).RemoteToolsBinDir(ServerArch::UseCase::kScp);
EXPECT_TRUE(absl::StrContains(win_scp_dir, "\\"));
EXPECT_TRUE(absl::StrContains(win_scp_dir, "AppData\\Roaming"));
std::string win_dir = ServerArch(kWindows).RemoteToolsBinDir();
EXPECT_TRUE(absl::StrContains(win_dir, "AppData\\Roaming\\"));
}
TEST(ServerArchTest, GetStartServerCommand) {
std::string cmd = ServerArch(kWindows).GetStartServerCommand(123, "foo bar");
EXPECT_TRUE(absl::StrContains(cmd, "123"));
EXPECT_TRUE(absl::StrContains(cmd, "foo bar"));
EXPECT_TRUE(absl::StrContains(cmd, "New-Item "));
EXPECT_TRUE(absl::StrContains(cmd, "cdc_rsync_server.exe foo bar"));
cmd = ServerArch(kLinux).GetStartServerCommand(123, "foo bar");
EXPECT_TRUE(absl::StrContains(cmd, "123"));
EXPECT_TRUE(absl::StrContains(cmd, "foo bar"));
EXPECT_TRUE(absl::StrContains(cmd, "mkdir -p"));
EXPECT_TRUE(absl::StrContains(cmd, "cdc_rsync_server foo bar"));
}
TEST(ServerArchTest, GetDeployReplaceCommand) {
std::string cmd = ServerArch(kWindows).GetDeployReplaceCommand("aaa", "bbb");
EXPECT_TRUE(absl::StrContains(cmd, "aaa"));
EXPECT_TRUE(absl::StrContains(cmd, "bbb"));
EXPECT_TRUE(absl::StrContains(cmd, "Move-Item "));
std::string cmd = ServerArch(kWindows).GetDeploySftpCommands();
EXPECT_TRUE(absl::StrContains(cmd, "cdc_rsync_server.exe"));
cmd = ServerArch(kLinux).GetDeployReplaceCommand("aaa", "bbb");
EXPECT_TRUE(absl::StrContains(cmd, "aaa"));
EXPECT_TRUE(absl::StrContains(cmd, "bbb"));
EXPECT_TRUE(absl::StrContains(cmd, "mv "));
cmd = ServerArch(kLinux).GetDeploySftpCommands();
EXPECT_TRUE(absl::StrContains(cmd, "cdc_rsync_server"));
}
} // namespace