github-mirrors/netris-cdc-file-transfer
1
0
Fork 0
mirror of https://github.com/nestriness/cdc-file-transfer.git synced 2026-01-30 14:25:36 +02:00
Code Issues Packages Projects Releases Wiki Activity

[cdc_rsync] [cdc_stream] Switch from scp to sftp (#66)

Browse Source 
Use sftp for deploying remote components instead of scp. sftp has the
advantage that it can also create directries, chmod files etc., so
that we can do everything in one call of sftp instead of mixing scp
and ssh calls.

The downside of sftp is that it can't switch to ~ resp. %userprofile%
for the remote side, and we have to assume that sftp starts in the
user's home dir. This is the default and works on my machines!

cdc_rsync and cdc_stream check the CDC_SFTP_COMMAND env var now and
accept --sftp-command flags. If they are not set, the corresponding
scp flag and env var is still used, with scp replaced by sftp. This is
most likely correct as sftp and scp usually reside in the same
directory and share largely identical parameters.
This commit is contained in:
Lutz Justen
2023-01-18 17:49:52 +01:00
committed by GitHub
parent a8b948b323
commit efca9855e7
19 changed files with 253 additions and 193 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
cdc_stream/asset_stream_config.cc
View File

@@ -157,9 +157,9 @@ void AssetStreamConfig::RegisterCommandLineFlags(lyra::command& cmd,
"connection to the host. See also --dev-src-dir."));
cmd.add_argument(
lyra::opt(dev_target_.scp_command, "cmd")
.name("--dev-scp-command")
.help("Scp command and extra flags to use for the "
lyra::opt(dev_target_.sftp_command, "cmd")
.name("--dev-sftp-command")
.help("Sftp command and extra flags to use for the "
"connection to the host. See also --dev-src-dir."));
cmd.add_argument(
@@ -258,7 +258,7 @@ std::string AssetStreamConfig::ToString() {
ss << "dev-user-host = " << dev_target_.user_host << std::endl;
ss << "dev-ssh-command = " << dev_target_.ssh_command
<< std::endl;
ss << "dev-scp-command = " << dev_target_.scp_command
ss << "dev-sftp-command = " << dev_target_.sftp_command
<< std::endl;
ss << "dev-mount-dir = " << dev_target_.mount_dir << std::endl;
return ss.str();

40
cdc_stream/cdc_fuse_manager.cc
View File

@@ -16,6 +16,7 @@
#include "absl/strings/match.h"
#include "absl/strings/str_format.h"
#include "absl/strings/str_split.h"
#include "cdc_fuse_fs/constants.h"
#include "common/gamelet_component.h"
#include "common/log.h"
@@ -30,7 +31,7 @@ constexpr char kExeFilename[] = "cdc_stream.exe";
constexpr char kFuseFilename[] = "cdc_fuse_fs";
constexpr char kLibFuseFilename[] = "libfuse.so";
constexpr char kFuseStdoutPrefix[] = "cdc_fuse_fs_stdout";
constexpr char kRemoteToolsBinDir[] = "~/.cache/cdc-file-transfer/bin/";
constexpr char kRemoteToolsBinDir[] = ".cache/cdc-file-transfer/bin/";
// Cache directory on the gamelet to store data chunks.
constexpr char kCacheDir[] = "~/.cache/cdc-file-transfer/chunks";
@@ -54,23 +55,25 @@ absl::Status CdcFuseManager::Deploy() {
std::string exe_dir;
RETURN_IF_ERROR(path::GetExeDir(&exe_dir), "Failed to get exe directory");
// Set the cwd to the exe dir and pass the filenames to scp. Otherwise, some
// scp implementations can get confused and create the wrong remote filenames.
path::SetCwd(exe_dir);
// Create the remote tools bin dir if it doesn't exist yet.
// This assumes that sftp's remote startup directory is the home directory.
std::vector<std::string> dir_parts =
absl::StrSplit(kRemoteToolsBinDir, '/', absl::SkipEmpty());
std::string sftp_commands;
for (const std::string& dir : dir_parts) {
// Use -mkdir to ignore errors if the directory already exists.
sftp_commands += absl::StrFormat("-mkdir %s\ncd %s\n", dir, dir);
}
// Copy FUSE to the gamelet.
LOG_DEBUG("Copying FUSE");
RETURN_IF_ERROR(remote_util_->Scp({kFuseFilename, kLibFuseFilename},
kRemoteToolsBinDir, /*compress=*/false),
"Failed to copy FUSE to gamelet");
LOG_DEBUG("Copying FUSE succeeded");
sftp_commands += absl::StrFormat("put %s\n", kFuseFilename);
sftp_commands += absl::StrFormat("put %s\n", kLibFuseFilename);
sftp_commands += absl::StrFormat("chmod 755 %s\n", kFuseFilename);
// Make FUSE executable. Note that sync does it automatically.
LOG_DEBUG("Making FUSE executable");
std::string remotePath = path::JoinUnix(kRemoteToolsBinDir, kFuseFilename);
RETURN_IF_ERROR(remote_util_->Chmod("a+x", remotePath),
"Failed to set executable flag on FUSE");
LOG_DEBUG("Making FUSE succeeded");
LOG_DEBUG("Deploying FUSE");
RETURN_IF_ERROR(
remote_util_->Sftp(sftp_commands, exe_dir, /*compress=*/false),
"Failed to deploy FUSE");
LOG_DEBUG("Deploying FUSE succeeded");
return absl::OkStatus();
}
@@ -104,14 +107,13 @@ absl::Status CdcFuseManager::Start(const std::string& mount_dir,
// Build the remote command.
std::string remotePath = path::JoinUnix(kRemoteToolsBinDir, kFuseFilename);
std::string remote_command = absl::StrFormat(
"mkdir -p %s; LD_LIBRARY_PATH=%s %s "
"LD_LIBRARY_PATH=%s %s "
"--instance=%s "
"--components=%s --port=%i --cache_dir=%s "
"--verbosity=%i --cleanup_timeout=%i --access_idle_timeout=%i --stats=%i "
"--check=%i --cache_capacity=%u -- -o allow_root -o ro -o nonempty -o "
"auto_unmount %s%s%s",
kRemoteToolsBinDir, kRemoteToolsBinDir, remotePath,
RemoteUtil::QuoteForSsh(instance_),
kRemoteToolsBinDir, remotePath, RemoteUtil::QuoteForSsh(instance_),
RemoteUtil::QuoteForSsh(component_args), remote_port, kCacheDir,
verbosity, cleanup_timeout_sec, access_idle_timeout_sec, enable_stats,
check, cache_capacity, debug ? "-d " : "", singlethreaded ? "-s " : "",

4
cdc_stream/local_assets_stream_manager_client.cc
View File

@@ -38,13 +38,13 @@ LocalAssetsStreamManagerClient::~LocalAssetsStreamManagerClient() = default;
absl::Status LocalAssetsStreamManagerClient::StartSession(
const std::string& src_dir, const std::string& user_host,
const std::string& mount_dir, const std::string& ssh_command,
const std::string& scp_command) {
const std::string& sftp_command) {
StartSessionRequest request;
request.set_workstation_directory(src_dir);
request.set_user_host(user_host);
request.set_mount_dir(mount_dir);
request.set_ssh_command(ssh_command);
request.set_scp_command(scp_command);
request.set_sftp_command(sftp_command);
grpc::ClientContext context;
StartSessionResponse response;

4
cdc_stream/local_assets_stream_manager_client.h
View File

@@ -43,12 +43,12 @@ class LocalAssetsStreamManagerClient {
// |user_host| is the Linux host, formatted as [user@:host].
// |mount_dir| is the Linux target directory to stream to.
// |ssh_command| is the ssh command and extra arguments to use.
// |scp_command| is the scp command and extra arguments to use.
// |sftp_command| is the sftp command and extra arguments to use.
absl::Status StartSession(const std::string& src_dir,
const std::string& user_host,
const std::string& mount_dir,
const std::string& ssh_command,
const std::string& scp_command);
const std::string& sftp_command);
// Stops the streaming session to the Linux target |user_host|:|mount_dir|.
// |user_host| is the Linux host, formatted as [user@:host].

6
cdc_stream/local_assets_stream_manager_service_impl.cc
View File

@@ -208,7 +208,7 @@ LocalAssetsStreamManagerServiceImpl::GetTargetForStadia(
SessionTarget target;
target.mount_dir = request.mount_dir();
target.ssh_command = request.ssh_command();
target.scp_command = request.scp_command();
target.sftp_command = request.sftp_command();
// Parse instance/project/org id.
if (!ParseInstanceName(request.gamelet_name(), instance_id, project_id,
@@ -223,7 +223,7 @@ LocalAssetsStreamManagerServiceImpl::GetTargetForStadia(
InitSsh(*instance_id, *project_id, *organization_id));
target.user_host = "cloudcast@" + instance_ip;
// Note: Port must be set with ssh_command (-p) and scp_command (-P).
// Note: Port must be set with ssh_command (-p) and sftp_command (-P).
return target;
}
@@ -233,7 +233,7 @@ SessionTarget LocalAssetsStreamManagerServiceImpl::GetTarget(
target.user_host = request.user_host();
target.mount_dir = request.mount_dir();
target.ssh_command = request.ssh_command();
target.scp_command = request.scp_command();
target.sftp_command = request.sftp_command();
*instance_id = absl::StrCat(target.user_host, ":", target.mount_dir);
return target;

4
cdc_stream/session.cc
View File

@@ -55,8 +55,8 @@ Session::Session(std::string instance_id, const SessionTarget& target,
if (!target.ssh_command.empty()) {
remote_util_.SetSshCommand(target.ssh_command);
}
if (!target.scp_command.empty()) {
remote_util_.SetScpCommand(target.scp_command);
if (!target.sftp_command.empty()) {
remote_util_.SetSftpCommand(target.sftp_command);
}
}

4
cdc_stream/session.h
View File

@@ -38,8 +38,8 @@ struct SessionTarget {
std::string user_host;
// Ssh command to use to connect to the remote target.
std::string ssh_command;
// Scp command to use to copy files to the remote target.
std::string scp_command;
// Sftp command to use to copy files to the remote target.
std::string sftp_command;
// Directory on the remote target where to mount the streamed directory.
std::string mount_dir;
};

41
cdc_stream/start_command.cc
View File

@@ -22,6 +22,7 @@
#include "common/log.h"
#include "common/path.h"
#include "common/process.h"
#include "common/remote_util.h"
#include "common/status_macros.h"
#include "common/stopwatch.h"
#include "common/util.h"
@@ -73,17 +74,28 @@ void StartCommand::RegisterCommandLineFlags(lyra::command& cmd) {
path::GetEnv("CDC_SSH_COMMAND", &ssh_command_).IgnoreError();
cmd.add_argument(
lyra::opt(ssh_command_, "ssh_command")
lyra::opt(ssh_command_, "cmd")
.name("--ssh-command")
.help("Path and arguments of ssh command to use, e.g. "
"\"C:\\path\\to\\ssh.exe -F config_file -p 1234\". Can also be "
"specified by the CDC_SSH_COMMAND environment variable."));
path::GetEnv("CDC_SCP_COMMAND", &scp_command_).IgnoreError();
path::GetEnv("CDC_SFTP_COMMAND", &sftp_command_).IgnoreError();
cmd.add_argument(
lyra::opt(scp_command_, "scp_command")
lyra::opt(sftp_command_, "cmd")
.name("--sftp-command")
.help(
"Path and arguments of sftp command to use, e.g. "
"\"C:\\path\\to\\sftp.exe -F config_file -P 1234\". Can also be "
"specified by the CDC_SFTP_COMMAND environment variable."));
path::GetEnv("CDC_SCP_COMMAND", &deprecated_scp_command_).IgnoreError();
cmd.add_argument(
lyra::opt(deprecated_scp_command_, "cmd")
.name("--scp-command")
.help("Path and arguments of scp command to use, e.g. "
.help("[Deprecated, use --sftp-command] Path and arguments of scp "
"command to "
"use, e.g. "
"\"C:\\path\\to\\scp.exe -F config_file -P 1234\". Can also be "
"specified by the CDC_SCP_COMMAND environment variable."));
@@ -106,9 +118,26 @@ absl::Status StartCommand::Run() {
RETURN_IF_ERROR(LocalAssetsStreamManagerClient::ParseUserHostDir(
user_host_dir_, &user_host, &mount_dir));
// Backwards compatibility after switching from scp to sftp.
if (sftp_command_.empty() && !deprecated_scp_command_.empty()) {
LOG_WARNING(
"The CDC_SCP_COMMAND environment variable and the --scp-command flag "
"are deprecated. Please set CDC_SFTP_COMMAND or --sftp-command "
"instead.");
sftp_command_ = RemoteUtil::ScpToSftpCommand(deprecated_scp_command_);
if (!sftp_command_.empty()) {
LOG_WARNING("Converted scp command '%s' to sftp command '%s'.",
deprecated_scp_command_, sftp_command_);
} else {
LOG_WARNING("Failed to convert scp command '%s' to sftp command.",
deprecated_scp_command_);
}
}
LocalAssetsStreamManagerClient client(CreateChannel(service_port_));
absl::Status status = client.StartSession(full_src_dir, user_host, mount_dir,
ssh_command_, scp_command_);
ssh_command_, sftp_command_);
if (absl::IsUnavailable(status)) {
LOG_DEBUG("StartSession status: %s", status.ToString());
@@ -122,7 +151,7 @@ absl::Status StartCommand::Run() {
// state.
LocalAssetsStreamManagerClient new_client(CreateChannel(service_port_));
status = new_client.StartSession(full_src_dir, user_host, mount_dir,
ssh_command_, scp_command_);
ssh_command_, sftp_command_);
}
}

4
cdc_stream/start_command.h
View File

@@ -44,9 +44,11 @@ class StartCommand : public BaseCommand {
int verbosity_ = 0;
uint16_t service_port_ = 0;
std::string ssh_command_;
std::string scp_command_;
std::string sftp_command_;
std::string src_dir_;
std::string user_host_dir_;
std::string deprecated_scp_command_;
};
} // namespace cdc_ft

2
cdc_stream/start_service_command.cc
View File

@@ -141,7 +141,7 @@ absl::Status StartServiceCommand::RunService() {
request.set_user_host(cfg_.dev_target().user_host);
request.set_mount_dir(cfg_.dev_target().mount_dir);
request.set_ssh_command(cfg_.dev_target().ssh_command);
request.set_scp_command(cfg_.dev_target().scp_command);
request.set_sftp_command(cfg_.dev_target().sftp_command);
localassetsstreammanager::StartSessionResponse response;
RETURN_ABSL_IF_ERROR(
session_service.StartSession(nullptr, &request, &response));