github-mirrors/netris-cdc-file-transfer
1
0
Fork 0
mirror of https://github.com/nestriness/cdc-file-transfer.git synced 2026-05-01 17:03:07 +03:00
Code Issues Packages Projects Releases Wiki Activity

[cdc_rsync] Detect remote architecture (#86)

Browse Source 
Improves ServerArch so that it can detect the remote architecture by
running uname and checking %PROCESSOR_ARCHITECTURE%. So far, only
x64 Linux and x64 Windows are supported, but in the future it is easy
to add support for others, e.g. aarch64, as well.

Before the detection is run, the remote architecture is guessed first
based on the destination. For instance, if the destination directory
starts with "C:\", it pretty much means Windows. If cdc_rsync_server
exists and runs fine, there's no need for detection.

Since also PortManager depends on the remote architecture, it has to
be adjusted as well. So far, PortManager assumeed that "local" means
Windows and "remote" means Linux. This is no longer the case for
syncing to Windows devices, so this CL adds the necessary abstractions
to PortManager.

Also refactors ArchType into a separate class in common, since it is
used now from several places. It is also expanded to handle future
changes that add support for different processor architectures, e.g.
aarch64.
This commit is contained in:
Lutz Justen
2023-02-01 11:51:20 +01:00
committed by GitHub
parent 3194678007
commit ee4118c6bf
19 changed files with 696 additions and 228 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
cdc_rsync/BUILD
View File

@@ -179,6 +179,7 @@ cc_library(
srcs = ["server_arch.cc"],
hdrs = ["server_arch.h"],
deps = [
"//common:arch_type",
"//common:path",
"//common:remote_util",
"@com_google_absl//absl/strings",

54
cdc_rsync/cdc_rsync_client.cc
View File

@@ -127,16 +127,35 @@ CdcRsyncClient::~CdcRsyncClient() {
}
absl::Status CdcRsyncClient::Run() {
int port;
ASSIGN_OR_RETURN(port, FindAvailablePort(), "Failed to find available port");
// If |remote_util_| is not set, it's a local sync. Otherwise, guess the
// architecture of the device that runs cdc_rsync_server from the destination
// path, e.g. "C:\path\to\dest" strongly indicates Windows.
ServerArch server_arch = !remote_util_
? ServerArch::DetectFromLocalDevice()
: ServerArch::GuessFromDestination(destination_);
// If |remote_util_| is not set, it's a local sync.
ServerArch::Type arch_type =
remote_util_ ? ServerArch::Detect(destination_) : ServerArch::LocalType();
ServerArch server_arch(arch_type);
int port;
ASSIGN_OR_RETURN(port, FindAvailablePort(&server_arch),
"Failed to find available port");
// Start the server process.
absl::Status status = StartServer(port, server_arch);
if (HasTag(status, Tag::kDeployServer) && server_arch.IsGuess() &&
server_exit_code_ != kServerExitCodeOutOfDate) {
// Server couldn't be run, e.g. not found or failed to start.
// Check whether we guessed the arch type wrong and try again.
// Note that in case of a local sync, or if the server actively reported
// that it's out-dated, there's no need to detect the arch.
const ArchType old_type = server_arch.GetType();
ASSIGN_OR_RETURN(server_arch,
ServerArch::DetectFromRemoteDevice(remote_util_.get()));
if (server_arch.GetType() != old_type) {
LOG_DEBUG("Guessed server arch type wrong, guessed %s, actual %s.",
GetArchTypeStr(old_type), server_arch.GetTypeStr());
status = StartServer(port, server_arch);
}
}
if (HasTag(status, Tag::kDeployServer)) {
// Gamelet components are not deployed or out-dated. Deploy and retry.
status = DeployServer(server_arch);
@@ -176,15 +195,17 @@ absl::Status CdcRsyncClient::Run() {
return status;
}
absl::StatusOr<int> CdcRsyncClient::FindAvailablePort() {
absl::StatusOr<int> CdcRsyncClient::FindAvailablePort(ServerArch* server_arch) {
// Find available local and remote ports for port forwarding.
// If only one port is in the given range, try that without checking.
if (options_.forward_port_first >= options_.forward_port_last) {
return options_.forward_port_first;
}
absl::StatusOr<int> port =
port_manager_->ReservePort(options_.connection_timeout_sec);
assert(server_arch);
absl::StatusOr<int> port = port_manager_->ReservePort(
options_.connection_timeout_sec, server_arch->GetType());
if (absl::IsDeadlineExceeded(port.status())) {
// Server didn't respond in time.
return SetTag(port.status(), Tag::kConnectionTimeout);
@@ -193,6 +214,21 @@ absl::StatusOr<int> CdcRsyncClient::FindAvailablePort() {
// Port in use.
return SetTag(port.status(), Tag::kAddressInUse);
}
// If |server_arch| was guessed, calling netstat might have failed because
// the arch was wrong. Properly detect it and try again if it changed.
if (!port.ok() && server_arch->IsGuess()) {
const ArchType old_type = server_arch->GetType();
ASSIGN_OR_RETURN(*server_arch,
ServerArch::DetectFromRemoteDevice(remote_util_.get()));
assert(!server_arch->IsGuess());
if (server_arch->GetType() != old_type) {
LOG_DEBUG("Guessed server arch type wrong, guessed %s, actual %s.",
GetArchTypeStr(old_type), server_arch->GetTypeStr());
return FindAvailablePort(server_arch);
}
}
return port;
}

4
cdc_rsync/cdc_rsync_client.h
View File

@@ -79,7 +79,9 @@ class CdcRsyncClient {
private:
// Finds available local and remote ports for port forwarding.
absl::StatusOr<int> FindAvailablePort();
// May update |server_arch| by properly detecting the architecture and retry
// if the architecture was guessed, i.e. if |server_arch|->IsGuess() is true.
absl::StatusOr<int> FindAvailablePort(ServerArch* server_arch);
// Starts the server process. If the method returns a status with tag
// |kTagDeployServer|, Run() calls DeployServer() and tries again.

201
cdc_rsync/server_arch.cc
View File

@@ -20,60 +20,152 @@
#include "absl/strings/str_format.h"
#include "absl/strings/str_split.h"
#include "common/path.h"
#include "common/platform.h"
#include "common/remote_util.h"
#include "common/status_macros.h"
#include "common/util.h"
namespace cdc_ft {
namespace {
constexpr char kErrorFailedToGetKnownFolderPath[] =
"error_failed_to_get_known_folder_path";
constexpr char kErrorArchTypeUnhandled[] = "arch_type_unhandled";
constexpr char kUnsupportedArchErrorFmt[] =
"Unsupported remote device architecture '%s'. If you think this is a "
"bug, or if this combination should be supported, please file a bug at "
"https://github.com/google/cdc-file-transfer.";
absl::StatusOr<ArchType> GetArchTypeFromUname(const std::string& uname_out) {
// uname_out is "KERNEL MACHINE"
// Possible values for KERNEL: Linux (not sure what else).
// Possible values for MACHINE:
// https://stackoverflow.com/questions/45125516/possible-values-for-uname-m
// Relevant for us: x86_64, aarch64.
if (absl::StartsWith(uname_out, "Linux ")) {
// Linux kernel. Check CPU type.
if (absl::StrContains(uname_out, "x86_64")) {
return ArchType::kLinux_x86_64;
}
}
if (absl::StartsWith(uname_out, "MSYS_")) {
// Windows machine that happens to have Cygwin/MSYS on it. Check CPU type.
if (absl::StrContains(uname_out, "x86_64")) {
return ArchType::kWindows_x86_64;
}
}
return absl::UnimplementedError(
absl::StrFormat(kUnsupportedArchErrorFmt, uname_out));
}
absl::StatusOr<ArchType> GetArchTypeFromWinProcArch(
const std::string& arch_out) {
// Possible values: AMD64, IA64, ARM64, x86
if (absl::StrContains(arch_out, "AMD64")) {
return ArchType::kWindows_x86_64;
}
return absl::UnimplementedError(
absl::StrFormat(kUnsupportedArchErrorFmt, arch_out));
}
} // namespace
// static
ServerArch::Type ServerArch::Detect(const std::string& destination) {
ServerArch ServerArch::GuessFromDestination(const std::string& destination) {
// Path starting with ~ or / -> Linux.
if (absl::StartsWith(destination, "~") ||
absl::StartsWith(destination, "/")) {
return Type::kLinux;
LOG_DEBUG("Guessed server arch type Linux based on ~ or /");
return ServerArch(ArchType::kLinux_x86_64, /*is_guess=*/true);
}
// Path starting with C: etc. -> Windows.
if (!path::GetDrivePrefix(destination).empty()) {
return Type::kWindows;
LOG_DEBUG("Guessed server arch type Windows based on drive prefix");
return ServerArch(ArchType::kWindows_x86_64, /*is_guess=*/true);
}
// Path with only / -> Linux.
if (absl::StrContains(destination, "/") &&
!absl::StrContains(destination, "\\")) {
return Type::kLinux;
LOG_DEBUG("Guessed server arch type Linux based on forward slashes");
return ServerArch(ArchType::kLinux_x86_64, /*is_guess=*/true);
}
// Path with only \\ -> Windows.
if (absl::StrContains(destination, "\\") &&
!absl::StrContains(destination, "/")) {
return Type::kWindows;
LOG_DEBUG("Guessed server arch type Windows based on backslashes");
return ServerArch(ArchType::kWindows_x86_64, /*is_guess=*/true);
}
// Default to Linux.
return Type::kLinux;
LOG_DEBUG("Guessed server arch type Linux as default");
return ServerArch(ArchType::kLinux_x86_64, /*is_guess=*/true);
}
// static
ServerArch::Type ServerArch::LocalType() {
#if PLATFORM_WINDOWS
return ServerArch::Type::kWindows;
#elif PLATFORM_LINUX
return ServerArch::Type::kLinux;
#endif
ServerArch ServerArch::DetectFromLocalDevice() {
LOG_DEBUG("Detected local device type %s",
GetArchTypeStr(GetLocalArchType()));
return ServerArch(GetLocalArchType(), /*is_guess=*/false);
}
// static
absl::StatusOr<ServerArch> ServerArch::DetectFromRemoteDevice(
RemoteUtil* remote_util) {
assert(remote_util);
// Run uname, assuming it's a Linux machine.
std::string uname_out;
std::string linux_cmd = "uname -sm";
absl::Status status =
remote_util->RunWithCapture(linux_cmd, "uname", &uname_out, nullptr);
if (status.ok()) {
LOG_DEBUG("Uname returned '%s'", uname_out);
absl::StatusOr<ArchType> type = GetArchTypeFromUname(uname_out);
if (type.ok()) {
LOG_DEBUG("Detected server arch type '%s' from uname",
GetArchTypeStr(*type));
return ServerArch(*type, /*is_guess=*/false);
}
status = type.status();
}
LOG_DEBUG(
"Failed to detect arch type from uname; this is expected if the remote "
"machine is not Linux; will try Windows next: %s",
status.ToString());
// Check %PROCESSOR_ARCHITECTURE%, assuming it's a Windows machine.
// Note: That space after PROCESSOR_ARCHITECTURE is important or else Windows
// command magic interprets quotes as part of the string.
std::string arch_out;
std::string windows_cmd =
RemoteUtil::QuoteForSsh("cmd /C set PROCESSOR_ARCHITECTURE ");
status = remote_util->RunWithCapture(
windows_cmd, "set PROCESSOR_ARCHITECTURE", &arch_out, nullptr);
if (status.ok()) {
LOG_DEBUG("PROCESSOR_ARCHITECTURE is '%s'", arch_out);
absl::StatusOr<ArchType> type = GetArchTypeFromWinProcArch(arch_out);
if (type.ok()) {
LOG_DEBUG("Detected server arch type '%s' from PROCESSOR_ARCHITECTURE",
GetArchTypeStr(*type));
return ServerArch(*type, /*is_guess=*/false);
}
status = type.status();
}
LOG_DEBUG("Failed to detect arch type from PROCESSOR_ARCHITECTURE: %s",
status.ToString());
return absl::InternalError("Failed to detect remote architecture");
}
// static
std::string ServerArch::CdcRsyncFilename() {
switch (LocalType()) {
case Type::kWindows:
switch (GetLocalArchType()) {
case ArchType::kWindows_x86_64:
return "cdc_rsync.exe";
case Type::kLinux:
case ArchType::kLinux_x86_64:
return "cdc_rsync";
default:
assert(!kErrorArchTypeUnhandled);
@@ -81,15 +173,18 @@ std::string ServerArch::CdcRsyncFilename() {
}
}
ServerArch::ServerArch(Type type) : type_(type) {}
ServerArch::ServerArch(ArchType type, bool is_guess)
: type_(type), is_guess_(is_guess) {}
ServerArch::~ServerArch() {}
const char* ServerArch::GetTypeStr() const { return GetArchTypeStr(type_); }
std::string ServerArch::CdcServerFilename() const {
switch (type_) {
case Type::kWindows:
case ArchType::kWindows_x86_64:
return "cdc_rsync_server.exe";
case Type::kLinux:
case ArchType::kLinux_x86_64:
return "cdc_rsync_server";
default:
assert(!kErrorArchTypeUnhandled);
@@ -98,46 +193,46 @@ std::string ServerArch::CdcServerFilename() const {
}
std::string ServerArch::RemoteToolsBinDir() const {
switch (type_) {
case Type::kWindows: {
return "AppData\\Roaming\\cdc-file-transfer\\bin\\";
}
case Type::kLinux:
return ".cache/cdc-file-transfer/bin/";
default:
assert(!kErrorArchTypeUnhandled);
return std::string();
if (IsWindowsArchType(type_)) {
return "AppData\\Roaming\\cdc-file-transfer\\bin\\";
}
if (IsLinuxArchType(type_)) {
return ".cache/cdc-file-transfer/bin/";
}
assert(!kErrorArchTypeUnhandled);
return std::string();
}
std::string ServerArch::GetStartServerCommand(int exit_code_not_found,
const std::string& args) const {
std::string server_path = RemoteToolsBinDir() + CdcServerFilename();
switch (type_) {
case Type::kWindows:
// TODO(ljusten): On Windows, ssh does not seem to forward the Powershell
// exit code (exit_code_not_found) to the process. However, that's really
// a minor issue and means we display "Deploying server..." instead of
// "Server not deployed. Deploying...";
return RemoteUtil::QuoteForWindows(
absl::StrFormat("powershell -Command \" "
"Set-StrictMode -Version 2; "
"$ErrorActionPreference = 'Stop'; "
"if (-not (Test-Path -Path '%s')) { "
" exit %i; "
"} "
"%s %s "
"\"",
server_path, exit_code_not_found, server_path, args));
case Type::kLinux:
return absl::StrFormat("if [ ! -f %s ]; then exit %i; fi; %s %s",
server_path, exit_code_not_found, server_path,
args);
default:
assert(!kErrorArchTypeUnhandled);
return std::string();
if (IsWindowsArchType(type_)) {
// TODO(ljusten): On Windows, ssh does not seem to forward the Powershell
// exit code (exit_code_not_found) to the process. However, that's really
// a minor issue and means we display "Deploying server..." instead of
// "Server not deployed. Deploying...";
return RemoteUtil::QuoteForWindows(
absl::StrFormat("powershell -Command \" "
"Set-StrictMode -Version 2; "
"$ErrorActionPreference = 'Stop'; "
"if (-not (Test-Path -Path '%s')) { "
" exit %i; "
"} "
"%s %s "
"\"",
server_path, exit_code_not_found, server_path, args));
}
if (IsLinuxArchType(type_)) {
return absl::StrFormat("if [ ! -f %s ]; then exit %i; fi; %s %s",
server_path, exit_code_not_found, server_path, args);
}
assert(!kErrorArchTypeUnhandled);
return std::string();
}
std::string ServerArch::GetDeploySftpCommands() const {

43
cdc_rsync/server_arch.h
View File

@@ -19,29 +19,49 @@
#include <string>
#include "absl/status/statusor.h"
#include "common/arch_type.h"
namespace cdc_ft {
class RemoteUtil;
// Abstracts all architecture specifics of cdc_rsync_server deployment.
// Comes in two flavors, "guessed" and "detected". Guesses are used as an
// optimization. For instance, if one syncs to C:\some\path, it's clearly a
// Windows machine and we can skip detection.
class ServerArch {
public:
enum class Type {
kLinux = 0,
kWindows = 1,
};
// Detects the arch type based on the destination path, e.g. path
// starting with C: indicate Windows.
static Type Detect(const std::string& destination);
// Guesses the arch type based on the destination path, e.g. path starting
// with C: indicate Windows. This is a guessed type. It may be wrong. For
// instance, if destination is just a single folder like "foo", the method
// defaults to Type::kLinux.
static ServerArch GuessFromDestination(const std::string& destination);
// Returns the arch type that matches the current process's type.
static Type LocalType();
// This is not a guessed type, it is reliable.
static ServerArch DetectFromLocalDevice();
// Creates an by properly detecting it on the remote device.
// This is more costly than guessing, but it is reliable.
static absl::StatusOr<ServerArch> DetectFromRemoteDevice(
RemoteUtil* remote_util);
// Returns the (local!) arch specific filename of cdc_rsync[.exe].
static std::string CdcRsyncFilename();
ServerArch(Type type);
ServerArch(ArchType type, bool is_guess);
~ServerArch();
// Accessor for the arch type.
ArchType GetType() const { return type_; }
// Returns the type as a human readable string.
const char* GetTypeStr() const;
// Returns true if the type was guessed and not detected.
bool IsGuess() const { return is_guess_; }
// Returns the arch-specific filename of cdc_rsync_server[.exe].
std::string CdcServerFilename() const;
@@ -68,7 +88,8 @@ class ServerArch {
std::string GetDeploySftpCommands() const;
private:
Type type_;
ArchType type_;
bool is_guess_ = false;
};
} // namespace cdc_ft

77
cdc_rsync/server_arch_test.cc
View File

@@ -20,68 +20,83 @@
namespace cdc_ft {
namespace {
constexpr auto kLinux = ServerArch::Type::kLinux;
constexpr auto kWindows = ServerArch::Type::kWindows;
constexpr auto kLinux = ArchType::kLinux_x86_64;
constexpr auto kWindows = ArchType::kWindows_x86_64;
TEST(ServerArchTest, DetectsLinuxIfPathStartsWithSlashOrTilde) {
EXPECT_EQ(ServerArch::Detect("/linux/path"), kLinux);
EXPECT_EQ(ServerArch::Detect("/linux\\path"), kLinux);
EXPECT_EQ(ServerArch::Detect("~/linux/path"), kLinux);
EXPECT_EQ(ServerArch::Detect("~/linux\\path"), kLinux);
EXPECT_EQ(ServerArch::Detect("~\\linux\\path"), kLinux);
constexpr bool kNoGuess = false;
TEST(ServerArchTest, GuessesLinuxIfPathStartsWithSlashOrTilde) {
EXPECT_EQ(ServerArch::GuessFromDestination("/linux/path").GetType(), kLinux);
EXPECT_EQ(ServerArch::GuessFromDestination("/linux\\path").GetType(), kLinux);
EXPECT_EQ(ServerArch::GuessFromDestination("~/linux/path").GetType(), kLinux);
EXPECT_EQ(ServerArch::GuessFromDestination("~/linux\\path").GetType(),
kLinux);
EXPECT_EQ(ServerArch::GuessFromDestination("~\\linux\\path").GetType(),
kLinux);
}
TEST(ServerArchTest, DetectsWindowsIfPathStartsWithDrive) {
EXPECT_EQ(ServerArch::Detect("C:\\win\\path"), kWindows);
EXPECT_EQ(ServerArch::Detect("D:win"), kWindows);
EXPECT_EQ(ServerArch::Detect("Z:\\win/path"), kWindows);
TEST(ServerArchTest, GuessesWindowsIfPathStartsWithDrive) {
EXPECT_EQ(ServerArch::GuessFromDestination("C:\\win\\path").GetType(),
kWindows);
EXPECT_EQ(ServerArch::GuessFromDestination("D:win").GetType(), kWindows);
EXPECT_EQ(ServerArch::GuessFromDestination("Z:\\win/path").GetType(),
kWindows);
}
TEST(ServerArchTest, DetectsLinuxIfPathOnlyHasForwardSlashes) {
EXPECT_EQ(ServerArch::Detect("linux/path"), kLinux);
TEST(ServerArchTest, GuessesLinuxIfPathOnlyHasForwardSlashes) {
EXPECT_EQ(ServerArch::GuessFromDestination("linux/path").GetType(), kLinux);
}
TEST(ServerArchTest, DetectsWindowsIfPathOnlyHasBackSlashes) {
EXPECT_EQ(ServerArch::Detect("\\win\\path"), kWindows);
TEST(ServerArchTest, GuessesWindowsIfPathOnlyHasBackSlashes) {
EXPECT_EQ(ServerArch::GuessFromDestination("\\win\\path").GetType(),
kWindows);
}
TEST(ServerArchTest, DetectsLinuxByDefault) {
EXPECT_EQ(ServerArch::Detect("/mixed\\path"), kLinux);
EXPECT_EQ(ServerArch::Detect("/mixed\\path"), kLinux);
EXPECT_EQ(ServerArch::Detect("C\\linux/path"), kLinux);
EXPECT_EQ(ServerArch::Detect(""), kLinux);
TEST(ServerArchTest, GuessesLinuxByDefault) {
EXPECT_EQ(ServerArch::GuessFromDestination("/mixed\\path").GetType(), kLinux);
EXPECT_EQ(ServerArch::GuessFromDestination("/mixed\\path").GetType(), kLinux);
EXPECT_EQ(ServerArch::GuessFromDestination("C\\linux/path").GetType(),
kLinux);
EXPECT_EQ(ServerArch::GuessFromDestination("").GetType(), kLinux);
}
TEST(ServerArchTest, IsGuess) {
EXPECT_TRUE(ServerArch::GuessFromDestination("foo").IsGuess());
EXPECT_FALSE(ServerArch::DetectFromLocalDevice().IsGuess());
}
TEST(ServerArchTest, CdcServerFilename) {
EXPECT_FALSE(
absl::StrContains(ServerArch(kLinux).CdcServerFilename(), "exe"));
EXPECT_TRUE(
absl::StrContains(ServerArch(kWindows).CdcServerFilename(), "exe"));
EXPECT_FALSE(absl::StrContains(
ServerArch(kLinux, kNoGuess).CdcServerFilename(), "exe"));
EXPECT_TRUE(absl::StrContains(
ServerArch(kWindows, kNoGuess).CdcServerFilename(), "exe"));
}
TEST(ServerArchTest, RemoteToolsBinDir) {
const std::string linux_dir = ServerArch(kLinux).RemoteToolsBinDir();
const std::string linux_dir =
ServerArch(kLinux, kNoGuess).RemoteToolsBinDir();
EXPECT_TRUE(absl::StrContains(linux_dir, ".cache/"));
std::string win_dir = ServerArch(kWindows).RemoteToolsBinDir();
std::string win_dir = ServerArch(kWindows, kNoGuess).RemoteToolsBinDir();
EXPECT_TRUE(absl::StrContains(win_dir, "AppData\\Roaming\\"));
}
TEST(ServerArchTest, GetStartServerCommand) {
std::string cmd = ServerArch(kWindows).GetStartServerCommand(123, "foo bar");
std::string cmd =
ServerArch(kWindows, kNoGuess).GetStartServerCommand(123, "foo bar");
EXPECT_TRUE(absl::StrContains(cmd, "123"));
EXPECT_TRUE(absl::StrContains(cmd, "cdc_rsync_server.exe foo bar"));
cmd = ServerArch(kLinux).GetStartServerCommand(123, "foo bar");
cmd = ServerArch(kLinux, kNoGuess).GetStartServerCommand(123, "foo bar");
EXPECT_TRUE(absl::StrContains(cmd, "123"));
EXPECT_TRUE(absl::StrContains(cmd, "cdc_rsync_server foo bar"));
}
TEST(ServerArchTest, GetDeployReplaceCommand) {
std::string cmd = ServerArch(kWindows).GetDeploySftpCommands();
std::string cmd = ServerArch(kWindows, kNoGuess).GetDeploySftpCommands();
EXPECT_TRUE(absl::StrContains(cmd, "cdc_rsync_server.exe"));
cmd = ServerArch(kLinux).GetDeploySftpCommands();
cmd = ServerArch(kLinux, kNoGuess).GetDeploySftpCommands();
EXPECT_TRUE(absl::StrContains(cmd, "cdc_rsync_server"));
}