diff --git a/NMakeBazelProject.targets b/NMakeBazelProject.targets index e194096..8c10aad 100644 --- a/NMakeBazelProject.targets +++ b/NMakeBazelProject.targets @@ -32,8 +32,7 @@ | sed -r "s/^([^:\(]+[:\(][[:digit:]]+(,[[:digit:]]+)?[:\)])/$(BazelSourcePathPrefix)\\1/" 2>&1 $(BazelSedCommand) - - --config=$(BazelPlatform) --workspace_status_command="exit 0" --bes_backend= + --config=$(BazelPlatform) $(BazelArgs) --linkopt=-Wl,--strip-all $(BazelArgs) --distinct_host_configuration=false diff --git a/README.md b/README.md index a0a3239..292901a 100644 --- a/README.md +++ b/README.md @@ -6,9 +6,113 @@ on Content Defined Chunking (CDC), in particular to split up files into chunks. ## CDC RSync -Tool to sync files to a remote machine, similar to the standard Linux -[rsync](https://linux.die.net/man/1/rsync). It supports fast compression and -uses a higher performing remote diffing approach based on CDC. -## Asset Streaming -Tool to stream assets from a Windows machine to a Linux device. \ No newline at end of file +CDC RSync is a tool to sync files from a Windows machine to a Linux device, +similar to the standard Linux [rsync](https://linux.die.net/man/1/rsync). It is +basically a copy tool, but optimized for the case where there is already an old +version of the files available in the target directory. +* It skips files quickly if timestamp and file size match. +* It uses fast compression for all data transfer. +* If a file changed, it determines which parts changed and only transfers the + differences. + +The remote diffing algorithm is based on CDC. In our tests, it is up to 30x +faster than the one used in rsync (1500 MB/s vs 50 MB/s). + +## CDC Stream + +CDC Stream is a tool to stream files and directories from a Windows machine to a +Linux device. Conceptually, it is similar to [sshfs](https://github.com/libfuse/sshfs), +but it is optimized for read speed. +* It caches streamed data on the Linux device. +* If a file is re-read on Linux after it changed on Windows, only the + differences are streamed again. The rest is read from cache. +* Stat operations are very fast since the directory metadata (filenames, + permissions etc.) is provided in a streaming-friendly way. + +To efficiently determine which parts of a file changed, the tool uses the same +CDC-based diffing algorithm as CDC RSync. Changes to Windows files are almost +immediately reflected on Linux, with a delay of roughly (0.5s + 0.7s x total +size of changed files in GB). + +The tool does not support writing files back from Linux to Windows; the Linux +directory is readonly. + +# Getting Started + +The project has to be built both on Windows and Linux. + +## Prerequisites + +The following steps have to be executed on **both Windows and Linux**. + +* Download and install Bazel from https://bazel.build/install. +* Clone the repository. + ``` + git clone https://github.com/google/cdc-file-transfer + ``` +* Initialize submodules. + ``` + cd cdc-file-transfer + git submodule update --init --recursive + ``` + +Finally, install an SSH client on the Windows device if not present. +The file transfer tools require `ssh.exe` and `scp.exe`. + +## Building + +The two tools can be built and used independently. + +### CDC Sync + +* Build Linux components + ``` + bazel build --config linux --compilation_mode=opt //cdc_rsync_server + ``` +* Build Windows components + ``` + bazel build --config windows --compilation_mode=opt //cdc_rsync + ``` +* Copy the Linux build output file `cdc_rsync_server` from + `bazel-bin/cdc_rsync_server` on the Linux system to `bazel-bin\cdc_rsync` + on the Windows machine. + +### CDC Stream + +* Build Linux components + ``` + bazel build --config linux --compilation_mode=opt //cdc_fuse_fs + ``` +* Build Windows components + ``` + bazel build --config windows --compilation_mode=opt //asset_stream_manager + ``` +* Copy the Linux build output files `cdc_fuse_fs` and `libfuse.so` from + `bazel-bin/cdc_fuse_fs` on the Linux system to `bazel-bin\asset_stream_manager` + on the Windows machine. + +## Usage + +### CDC Sync +To copy the contents of the Windows directory `C:\path\to\assets` to `~/assets` +on the Linux device `linux.machine.com`, run +``` +cdc_rsync --ssh-command=C:\path\to\ssh.exe --scp-command=C:\path\to\scp.exe C:\path\to\assets\* user@linux.machine.com:~/assets -vr +``` +Depending on your setup, you may have to specify additional arguments for the +ssh and scp commands, including proper quoting, e.g. +``` +cdc_rsync --ssh-command="\"C:\path with space\to\ssh.exe\" -F ssh_config_file -i id_rsa_file -oStrictHostKeyChecking=yes -oUserKnownHostsFile=\"\"\"known_hosts_file\"\"\"" --scp-command="\"C:\path with space\to\scp.exe\" -F ssh_config_file -i id_rsa_file -oStrictHostKeyChecking=yes -oUserKnownHostsFile=\"\"\"known_hosts_file\"\"\"" C:\path\to\assets\* user@linux.machine.com:~/assets -vr +``` +Lengthy ssh/scp commands that rarely change can also be put into environment +variables `CDC_SSH_COMMAND` and `CDC_SCP_COMMAND`, e.g. +``` +set CDC_SSH_COMMAND="C:\path with space\to\ssh.exe" -F ssh_config_file -i id_rsa_file -oStrictHostKeyChecking=yes -oUserKnownHostsFile="""known_hosts_file""" + +set CDC_SCP_COMMAND="C:\path with space\to\scp.exe" -F ssh_config_file -i id_rsa_file -oStrictHostKeyChecking=yes -oUserKnownHostsFile="""known_hosts_file""" + +cdc_rsync C:\path\to\assets\* user@linux.machine.com:~/assets -vr +``` + +### CDC Stream diff --git a/all_files.vcxitems b/all_files.vcxitems index 9428041..4dc5840 100644 --- a/all_files.vcxitems +++ b/all_files.vcxitems @@ -96,10 +96,8 @@ - - @@ -107,9 +105,9 @@ - - - + + + @@ -202,14 +200,12 @@ - - - + @@ -247,7 +243,6 @@ - @@ -259,6 +254,7 @@ + diff --git a/asset_stream_manager/cdc_fuse_manager.cc b/asset_stream_manager/cdc_fuse_manager.cc index a43bb23..e4cd4cc 100644 --- a/asset_stream_manager/cdc_fuse_manager.cc +++ b/asset_stream_manager/cdc_fuse_manager.cc @@ -29,7 +29,7 @@ namespace { constexpr char kFuseFilename[] = "cdc_fuse_fs"; constexpr char kLibFuseFilename[] = "libfuse.so"; constexpr char kFuseStdoutPrefix[] = "cdc_fuse_fs_stdout"; -constexpr char kRemoteToolsBinDir[] = "/opt/developer/tools/bin/"; +constexpr char kRemoteToolsBinDir[] = "~/.cache/cdc_file_transfer/"; // Mount point for FUSE on the gamelet. constexpr char kMountDir[] = "/mnt/workstation"; diff --git a/asset_stream_manager/session.cc b/asset_stream_manager/session.cc index 9666aae..77dcab8 100644 --- a/asset_stream_manager/session.cc +++ b/asset_stream_manager/session.cc @@ -51,7 +51,7 @@ Session::Session(std::string instance_id, std::string instance_ip, /*forward_output_to_logging=*/true), metrics_recorder_(std::move(metrics_recorder)) { assert(metrics_recorder_); - remote_util_.SetIpAndPort(instance_ip, instance_port); + remote_util_.SetUserHostAndPort(instance_ip, instance_port); } Session::~Session() { diff --git a/cdc_rsync/BUILD b/cdc_rsync/BUILD index 1c01242..f0ffa54 100644 --- a/cdc_rsync/BUILD +++ b/cdc_rsync/BUILD @@ -1,12 +1,16 @@ -load( - "//tools:windows_cc_library.bzl", - "cc_windows_shared_library", -) - package(default_visibility = [ "//:__subpackages__", ]) +cc_binary( + name = "cdc_rsync", + srcs = ["main.cc"], + deps = [ + ":cdc_rsync_client", + ":params", + ], +) + cc_library( name = "client_file_info", hdrs = ["client_file_info.h"], @@ -57,25 +61,16 @@ cc_test( ], ) -cc_windows_shared_library( - name = "cdc_rsync", - srcs = [ - "cdc_rsync.cc", - "cdc_rsync_client.cc", - "dllmain.cc", - ], - hdrs = [ - "cdc_rsync.h", - "cdc_rsync_client.h", - "error_messages.h", - ], +cc_library( + name = "cdc_rsync_client", + srcs = ["cdc_rsync_client.cc"], + hdrs = ["cdc_rsync_client.h"], linkopts = select({ "//tools:windows": [ "/DEFAULTLIB:Ws2_32.lib", # Sockets, e.g. recv, send, WSA*. ], "//conditions:default": [], }), - local_defines = ["COMPILING_DLL"], target_compatible_with = ["@platforms//os:windows"], deps = [ ":client_socket", @@ -128,6 +123,28 @@ cc_test( ], ) +cc_library( + name = "params", + srcs = ["params.cc"], + hdrs = ["params.h"], + deps = [ + ":cdc_rsync_client", + "@com_github_zstd//:zstd", + "@com_google_absl//absl/status", + ], +) + +cc_test( + name = "params_test", + srcs = ["params_test.cc"], + data = ["testdata/root.txt"] + glob(["testdata/params/**"]), + deps = [ + ":params", + "//common:test_main", + "@com_google_googletest//:gtest", + ], +) + cc_library( name = "progress_tracker", srcs = ["progress_tracker.cc"], diff --git a/cdc_rsync/cdc_rsync.cc b/cdc_rsync/cdc_rsync.cc deleted file mode 100644 index 1e2d279..0000000 --- a/cdc_rsync/cdc_rsync.cc +++ /dev/null @@ -1,125 +0,0 @@ -// Copyright 2022 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "cdc_rsync/cdc_rsync.h" - -#include - -#include "cdc_rsync/cdc_rsync_client.h" -#include "cdc_rsync/error_messages.h" -#include "common/log.h" -#include "common/path_filter.h" -#include "common/status.h" - -namespace cdc_ft { -namespace { - -ReturnCode TagToMessage(Tag tag, const Options* options, std::string* msg) { - msg->clear(); - switch (tag) { - case Tag::kSocketEof: - *msg = kMsgConnectionLost; - return ReturnCode::kConnectionLost; - - case Tag::kAddressInUse: - *msg = kMsgAddressInUse; - return ReturnCode::kAddressInUse; - - case Tag::kDeployServer: - *msg = kMsgDeployFailed; - return ReturnCode::kDeployFailed; - - case Tag::kInstancePickerNotAvailableInQuietMode: - *msg = kMsgInstancePickerNotAvailableInQuietMode; - return ReturnCode::kInstancePickerNotAvailableInQuietMode; - - case Tag::kConnectionTimeout: - *msg = - absl::StrFormat(kMsgFmtConnectionTimeout, options->ip, options->port); - return ReturnCode::kConnectionTimeout; - - case Tag::kCount: - return ReturnCode::kGenericError; - } - - // Should not happen (TM). Will fall back to status message in this case. - return ReturnCode::kGenericError; -} - -PathFilter::Rule::Type ToInternalType(FilterRule::Type type) { - switch (type) { - case FilterRule::Type::kInclude: - return PathFilter::Rule::Type::kInclude; - case FilterRule::Type::kExclude: - return PathFilter::Rule::Type::kExclude; - } - assert(false); - return PathFilter::Rule::Type::kInclude; -} - -} // namespace - -ReturnCode Sync(const Options* options, const FilterRule* filter_rules, - size_t num_filter_rules, const char* sources_dir, - const char* const* sources, size_t num_sources, - const char* destination, const char** error_message) { - LogLevel log_level = Log::VerbosityToLogLevel(options->verbosity); - Log::Initialize(std::make_unique(log_level)); - - PathFilter path_filter; - for (size_t n = 0; n < num_filter_rules; ++n) { - path_filter.AddRule(ToInternalType(filter_rules[n].type), - filter_rules[n].pattern); - } - - std::vector sources_vec; - for (size_t n = 0; n < num_sources; ++n) { - sources_vec.push_back(sources[n]); - } - - // Run rsync. - GgpRsyncClient client(*options, std::move(path_filter), sources_dir, - std::move(sources_vec), destination); - absl::Status status = client.Run(); - - if (status.ok()) { - *error_message = nullptr; - return ReturnCode::kOk; - } - - std::string msg; - ReturnCode code = ReturnCode::kGenericError; - absl::optional tag = GetTag(status); - if (tag.has_value()) { - code = TagToMessage(tag.value(), options, &msg); - } - - // Fall back to status message. - if (msg.empty()) { - msg = std::string(status.message()); - } else if (options->verbosity >= 2) { - // In verbose mode, log the status as well, so nothing gets lost. - LOG_ERROR("%s", status.ToString().c_str()); - } - - // Store error message in static buffer (don't use std::string through DLL - // boundary!). - static char buf[1024] = {0}; - strncpy_s(buf, msg.c_str(), _TRUNCATE); - *error_message = buf; - - return code; -} - -} // namespace cdc_ft diff --git a/cdc_rsync/cdc_rsync.h b/cdc_rsync/cdc_rsync.h deleted file mode 100644 index 9a13328..0000000 --- a/cdc_rsync/cdc_rsync.h +++ /dev/null @@ -1,107 +0,0 @@ -/* - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef CDC_RSYNC_CDC_RSYNC_H_ -#define CDC_RSYNC_CDC_RSYNC_H_ - -#ifdef COMPILING_DLL -#define CDC_RSYNC_API __declspec(dllexport) -#else -#define CDC_RSYNC_API __declspec(dllimport) -#endif - -namespace cdc_ft { - -#ifdef __cplusplus -extern "C" { -#endif - -struct Options { - const char* ip = nullptr; - int port = 0; - bool delete_ = false; - bool recursive = false; - int verbosity = 0; - bool quiet = false; - bool whole_file = false; - bool relative = false; - bool compress = false; - bool checksum = false; - bool dry_run = false; - bool existing = false; - bool json = false; - const char* copy_dest = nullptr; - int compress_level = 6; - int connection_timeout_sec = 10; - - // Compression level 0 is invalid. - static constexpr int kMinCompressLevel = -5; - static constexpr int kMaxCompressLevel = 22; -}; - -// Rule for including/excluding files. -struct FilterRule { - enum class Type { - kInclude, - kExclude, - }; - - Type type; - const char* pattern; - - FilterRule(Type type, const char* pattern) : type(type), pattern(pattern) {} -}; - -enum class ReturnCode { - // No error. Will match the tool's exit code, so OK must be 0. - kOk = 0, - - // Generic error. - kGenericError = 1, - - // Server connection timed out. - kConnectionTimeout = 2, - - // Connection to the server was shut down unexpectedly. - kConnectionLost = 3, - - // Binding to the forward port failed, probably because there's another - // instance of cdc_rsync running. - kAddressInUse = 4, - - // Server deployment failed. This should be rare, it means that the server - // components were successfully copied, but the up-to-date check still fails. - kDeployFailed = 5, - - // Gamelet selection asks for user input, but we are in quiet mode. - kInstancePickerNotAvailableInQuietMode = 6, -}; - -// Calling Sync() a second time overwrites the data in |error_message|. -CDC_RSYNC_API ReturnCode Sync(const Options* options, - const FilterRule* filter_rules, - size_t filter_num_rules, const char* sources_dir, - const char* const* sources, size_t num_sources, - const char* destination, - const char** error_message); - -#ifdef __cplusplus -} // extern "C" -#endif - -} // namespace cdc_ft - -#endif // CDC_RSYNC_CDC_RSYNC_H_ diff --git a/cdc_rsync_cli/cdc_rsync_cli.vcxproj b/cdc_rsync/cdc_rsync.vcxproj similarity index 98% rename from cdc_rsync_cli/cdc_rsync_cli.vcxproj rename to cdc_rsync/cdc_rsync.vcxproj index b0603e0..c83069a 100644 --- a/cdc_rsync_cli/cdc_rsync_cli.vcxproj +++ b/cdc_rsync/cdc_rsync.vcxproj @@ -42,12 +42,12 @@ - $(SolutionDir)bazel-out\x64_windows-dbg\bin\cdc_rsync_cli\ + $(SolutionDir)bazel-out\x64_windows-dbg\bin\cdc_rsync\ /std:c++17 UNICODE - $(SolutionDir)bazel-out\x64_windows-opt\bin\cdc_rsync_cli\ + $(SolutionDir)bazel-out\x64_windows-opt\bin\cdc_rsync\ UNICODE /std:c++17 @@ -66,7 +66,7 @@ - //cdc_rsync_cli:cdc_rsync + //cdc_rsync cdc_rsync.exe ..\;..\third_party\absl;..\third_party\blake3\c;..\bazel-stadia-file-transfer\external\com_github_zstd\lib;..\third_party\googletest\googletest\include;..\third_party\protobuf\src;$(VC_IncludePath);$(WindowsSDK_IncludePath) ..\/ diff --git a/cdc_rsync_cli/cdc_rsync_cli.vcxproj.filters b/cdc_rsync/cdc_rsync.vcxproj.filters similarity index 100% rename from cdc_rsync_cli/cdc_rsync_cli.vcxproj.filters rename to cdc_rsync/cdc_rsync.vcxproj.filters diff --git a/cdc_rsync/cdc_rsync_client.cc b/cdc_rsync/cdc_rsync_client.cc index c121af1..ea04cab 100644 --- a/cdc_rsync/cdc_rsync_client.cc +++ b/cdc_rsync/cdc_rsync_client.cc @@ -47,7 +47,7 @@ constexpr int kExitCodeNotFound = 127; constexpr int kForwardPortFirst = 44450; constexpr int kForwardPortLast = 44459; constexpr char kGgpServerFilename[] = "cdc_rsync_server"; -constexpr char kRemoteToolsBinDir[] = "/opt/developer/tools/bin/"; +constexpr char kRemoteToolsBinDir[] = "~/.cache/cdc_file_transfer/"; SetOptionsRequest::FilterRule::Type ToProtoType(PathFilter::Rule::Type type) { switch (type) { @@ -94,14 +94,12 @@ absl::Status GetServerExitStatus(int exit_code, const std::string& error_msg) { } // namespace -GgpRsyncClient::GgpRsyncClient(const Options& options, PathFilter path_filter, - std::string sources_dir, +CdcRsyncClient::CdcRsyncClient(const Options& options, std::vector sources, - std::string destination) + std::string user_host, std::string destination) : options_(options), - path_filter_(std::move(path_filter)), - sources_dir_(std::move(sources_dir)), sources_(std::move(sources)), + user_host_(std::move(user_host)), destination_(std::move(destination)), remote_util_(options.verbosity, options.quiet, &process_factory_, /*forward_output_to_log=*/false), @@ -109,24 +107,26 @@ GgpRsyncClient::GgpRsyncClient(const Options& options, PathFilter path_filter, kForwardPortFirst, kForwardPortLast, &process_factory_, &remote_util_), printer_(options.quiet, Util::IsTTY() && !options.json), - progress_(&printer_, options.verbosity, options.json) {} + progress_(&printer_, options.verbosity, options.json) { + if (!options_.ssh_command.empty()) { + remote_util_.SetSshCommand(options_.ssh_command); + } + if (!options_.scp_command.empty()) { + remote_util_.SetScpCommand(options_.scp_command); + } +} -GgpRsyncClient::~GgpRsyncClient() { +CdcRsyncClient::~CdcRsyncClient() { message_pump_.StopMessagePump(); socket_.Disconnect(); } -absl::Status GgpRsyncClient::Run() { - absl::Status status = remote_util_.GetInitStatus(); - if (!status.ok()) { - return WrapStatus(status, "Failed to initialize critical components"); - } - +absl::Status CdcRsyncClient::Run() { // Initialize |remote_util_|. - remote_util_.SetIpAndPort(options_.ip, options_.port); + remote_util_.SetUserHostAndPort(user_host_, options_.port); // Start the server process. - status = StartServer(); + absl::Status status = StartServer(); if (HasTag(status, Tag::kDeployServer)) { // Gamelet components are not deployed or out-dated. Deploy and retry. status = DeployServer(); @@ -166,7 +166,7 @@ absl::Status GgpRsyncClient::Run() { return status; } -absl::Status GgpRsyncClient::StartServer() { +absl::Status CdcRsyncClient::StartServer() { assert(!server_process_); // Components are expected to reside in the same dir as the executable. @@ -187,8 +187,8 @@ absl::Status GgpRsyncClient::StartServer() { std::string component_args = GameletComponent::ToCommandLineArgs(components); // Find available local and remote ports for port forwarding. - absl::StatusOr port_res = - port_manager_.ReservePort(options_.connection_timeout_sec); + absl::StatusOr port_res = port_manager_.ReservePort( + /*check_remote=*/false, /*remote_timeout_sec unused*/ 0); constexpr char kErrorMsg[] = "Failed to find available port"; if (absl::IsDeadlineExceeded(port_res.status())) { // Server didn't respond in time. @@ -205,9 +205,11 @@ absl::Status GgpRsyncClient::StartServer() { std::string(kRemoteToolsBinDir) + kGgpServerFilename; // Test existence manually to prevent misleading bash output message // "bash: .../cdc_rsync_server: No such file or directory". - std::string remote_command = absl::StrFormat( - "if [ ! -f %s ]; then exit %i; fi; %s %i %s", remote_server_path, - kExitCodeNotFound, remote_server_path, port, component_args); + // Also create the bin dir because otherwise scp below might fail. + std::string remote_command = + absl::StrFormat("mkdir -p %s; if [ ! -f %s ]; then exit %i; fi; %s %i %s", + kRemoteToolsBinDir, remote_server_path, kExitCodeNotFound, + remote_server_path, port, component_args); ProcessStartInfo start_info = remote_util_.BuildProcessStartInfoForSshPortForwardAndCommand( port, port, false, remote_command); @@ -225,16 +227,25 @@ absl::Status GgpRsyncClient::StartServer() { } // Wait until the server process is listening. - auto detect_listening = [is_listening = &is_server_listening_]() -> bool { - return *is_listening; + Stopwatch timeout_timer; + bool is_timeout = false; + auto detect_listening_or_timeout = [is_listening = &is_server_listening_, + timeout = options_.connection_timeout_sec, + &timeout_timer, &is_timeout]() -> bool { + is_timeout = timeout_timer.ElapsedSeconds() > timeout; + return *is_listening || is_timeout; }; - status = process->RunUntil(detect_listening); + status = process->RunUntil(detect_listening_or_timeout); if (!status.ok()) { // Some internal process error. Note that this does NOT mean that // cdc_rsync_server does not exist. In that case, the ssh process exits with // code 127. return status; } + if (is_timeout) { + return SetTag(absl::DeadlineExceededError("Timeout while starting server"), + Tag::kConnectionTimeout); + } if (process->HasExited()) { // Don't re-deploy for code > kServerExitCodeOutOfDate, which means that the @@ -263,7 +274,7 @@ absl::Status GgpRsyncClient::StartServer() { return absl::OkStatus(); } -absl::Status GgpRsyncClient::StopServer() { +absl::Status CdcRsyncClient::StopServer() { assert(server_process_); // Close socket. @@ -282,7 +293,7 @@ absl::Status GgpRsyncClient::StopServer() { return absl::OkStatus(); } -absl::Status GgpRsyncClient::HandleServerOutput(const char* data) { +absl::Status CdcRsyncClient::HandleServerOutput(const char* data) { // Note: This is called from a background thread! // Handle server error messages. Unfortunately, if the server prints to @@ -319,7 +330,7 @@ absl::Status GgpRsyncClient::HandleServerOutput(const char* data) { return absl::OkStatus(); } -absl::Status GgpRsyncClient::Sync() { +absl::Status CdcRsyncClient::Sync() { absl::Status status = SendOptions(); if (!status.ok()) { return WrapStatus(status, "Failed to send options to server"); @@ -377,7 +388,7 @@ absl::Status GgpRsyncClient::Sync() { return status; } -absl::Status GgpRsyncClient::DeployServer() { +absl::Status CdcRsyncClient::DeployServer() { assert(!server_process_); std::string exe_dir; @@ -409,34 +420,26 @@ absl::Status GgpRsyncClient::DeployServer() { return WrapStatus(status, "Failed to copy cdc_rsync_server to instance"); } - // Make cdc_rsync_server executable. - status = remote_util_.Chmod("a+x", remoteServerTmpPath); + // Do 3 things in one SSH command, to save time: + // - Make the old cdc_rsync_server writable (if it exists). + // - Make the new cdc_rsync_server executable. + // - Replace the old cdc_rsync_server by the new one. + std::string old_path = RemoteUtil::EscapeForWindows( + std::string(kRemoteToolsBinDir) + kGgpServerFilename); + std::string new_path = RemoteUtil::EscapeForWindows(remoteServerTmpPath); + std::string replace_cmd = absl::StrFormat( + " ([ ! -f %s ] || chmod u+w %s) && chmod a+x %s && mv %s %s", old_path, + old_path, new_path, new_path, old_path); + status = remote_util_.Run(replace_cmd, "chmod && chmod && mv"); if (!status.ok()) { return WrapStatus(status, - "Failed to set executable flag on cdc_rsync_server"); - } - - // Make old file writable. Mv might fail to overwrite it, e.g. if someone made - // it read-only. - std::string remoteServerPath = - std::string(kRemoteToolsBinDir) + kGgpServerFilename; - status = remote_util_.Chmod("u+w", remoteServerPath, /*quiet=*/true); - if (!status.ok()) { - LOG_DEBUG("chmod u+w %s failed (expected if file does not exist): %s", - remoteServerPath, status.ToString()); - } - - // Replace old file by new file. - status = remote_util_.Mv(remoteServerTmpPath, remoteServerPath); - if (!status.ok()) { - return WrapStatus(status, "Failed to replace '%s' by '%s'", - remoteServerPath, remoteServerTmpPath); + "Failed to replace old cdc_rsync_server by new one"); } return absl::OkStatus(); } -absl::Status GgpRsyncClient::SendOptions() { +absl::Status CdcRsyncClient::SendOptions() { LOG_INFO("Sending options"); SetOptionsRequest request; @@ -448,7 +451,7 @@ absl::Status GgpRsyncClient::SendOptions() { request.set_compress(options_.compress); request.set_relative(options_.relative); - for (const PathFilter::Rule& rule : path_filter_.GetRules()) { + for (const PathFilter::Rule& rule : options_.filter.GetRules()) { SetOptionsRequest::FilterRule* filter_rule = request.add_filter_rules(); filter_rule->set_type(ToProtoType(rule.type)); filter_rule->set_pattern(rule.pattern); @@ -457,7 +460,7 @@ absl::Status GgpRsyncClient::SendOptions() { request.set_checksum(options_.checksum); request.set_dry_run(options_.dry_run); request.set_existing(options_.existing); - if (options_.copy_dest) { + if (!options_.copy_dest.empty()) { request.set_copy_dest(options_.copy_dest); } @@ -470,13 +473,13 @@ absl::Status GgpRsyncClient::SendOptions() { return absl::OkStatus(); } -absl::Status GgpRsyncClient::FindAndSendAllSourceFiles() { +absl::Status CdcRsyncClient::FindAndSendAllSourceFiles() { LOG_INFO("Finding and sending all sources files"); Stopwatch stopwatch; - FileFinderAndSender file_finder(&path_filter_, &message_pump_, &progress_, - sources_dir_, options_.recursive, + FileFinderAndSender file_finder(&options_.filter, &message_pump_, &progress_, + options_.sources_dir, options_.recursive, options_.relative); progress_.StartFindFiles(); @@ -497,7 +500,7 @@ absl::Status GgpRsyncClient::FindAndSendAllSourceFiles() { return absl::OkStatus(); } -absl::Status GgpRsyncClient::ReceiveFileStats() { +absl::Status CdcRsyncClient::ReceiveFileStats() { LOG_INFO("Receiving file stats"); SendFileStatsResponse response; @@ -517,7 +520,7 @@ absl::Status GgpRsyncClient::ReceiveFileStats() { return absl::OkStatus(); } -absl::Status GgpRsyncClient::ReceiveDeletedFiles() { +absl::Status CdcRsyncClient::ReceiveDeletedFiles() { LOG_INFO("Receiving path of deleted files"); std::string current_directory; @@ -548,7 +551,7 @@ absl::Status GgpRsyncClient::ReceiveDeletedFiles() { return absl::OkStatus(); } -absl::Status GgpRsyncClient::ReceiveFileIndices( +absl::Status CdcRsyncClient::ReceiveFileIndices( const char* file_type, std::vector* file_indices) { LOG_INFO("Receiving indices of %s files", file_type); @@ -582,7 +585,7 @@ absl::Status GgpRsyncClient::ReceiveFileIndices( return absl::OkStatus(); } -absl::Status GgpRsyncClient::SendMissingFiles() { +absl::Status CdcRsyncClient::SendMissingFiles() { if (missing_file_indices_.empty()) { return absl::OkStatus(); } @@ -653,7 +656,7 @@ absl::Status GgpRsyncClient::SendMissingFiles() { return absl::OkStatus(); } -absl::Status GgpRsyncClient::ReceiveSignaturesAndSendDelta() { +absl::Status CdcRsyncClient::ReceiveSignaturesAndSendDelta() { if (changed_file_indices_.empty()) { return absl::OkStatus(); } @@ -731,7 +734,7 @@ absl::Status GgpRsyncClient::ReceiveSignaturesAndSendDelta() { return absl::OkStatus(); } -absl::Status GgpRsyncClient::StartCompressionStream() { +absl::Status CdcRsyncClient::StartCompressionStream() { assert(!compression_stream_); // Notify server that data is compressed from now on. @@ -762,7 +765,7 @@ absl::Status GgpRsyncClient::StartCompressionStream() { return absl::OkStatus(); } -absl::Status GgpRsyncClient::StopCompressionStream() { +absl::Status CdcRsyncClient::StopCompressionStream() { assert(compression_stream_); // Finish writing to |compression_process_|'s stdin and change back to diff --git a/cdc_rsync/cdc_rsync_client.h b/cdc_rsync/cdc_rsync_client.h index 20203ae..e9680d2 100644 --- a/cdc_rsync/cdc_rsync_client.h +++ b/cdc_rsync/cdc_rsync_client.h @@ -22,7 +22,6 @@ #include "absl/status/status.h" #include "cdc_rsync/base/message_pump.h" -#include "cdc_rsync/cdc_rsync.h" #include "cdc_rsync/client_socket.h" #include "cdc_rsync/progress_tracker.h" #include "common/path_filter.h" @@ -34,13 +33,38 @@ namespace cdc_ft { class Process; class ZstdStream; -class GgpRsyncClient { +class CdcRsyncClient { public: - GgpRsyncClient(const Options& options, PathFilter filter, - std::string sources_dir, std::vector sources, - std::string destination); + struct Options { + int port = RemoteUtil::kDefaultSshPort; + bool delete_ = false; + bool recursive = false; + int verbosity = 0; + bool quiet = false; + bool whole_file = false; + bool relative = false; + bool compress = false; + bool checksum = false; + bool dry_run = false; + bool existing = false; + bool json = false; + std::string copy_dest; + int compress_level = 6; + int connection_timeout_sec = 10; + std::string ssh_command; + std::string scp_command; + std::string sources_dir; // Base dir for files loaded for --files-from. + PathFilter filter; - ~GgpRsyncClient(); + // Compression level 0 is invalid. + static constexpr int kMinCompressLevel = -5; + static constexpr int kMaxCompressLevel = 22; + }; + + CdcRsyncClient(const Options& options, std::vector sources, + std::string user_host, std::string destination); + + ~CdcRsyncClient(); // Deploys the server if necessary, starts it and runs the rsync procedure. absl::Status Run(); @@ -93,11 +117,9 @@ class GgpRsyncClient { absl::Status StopCompressionStream(); Options options_; - PathFilter path_filter_; - const std::string sources_dir_; std::vector sources_; + const std::string user_host_; const std::string destination_; - WinProcessFactory process_factory_; RemoteUtil remote_util_; PortManager port_manager_; diff --git a/cdc_rsync/dllmain.cc b/cdc_rsync/dllmain.cc deleted file mode 100644 index 337cc93..0000000 --- a/cdc_rsync/dllmain.cc +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright 2022 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#define WIN32_LEAN_AND_MEAN -#include - -BOOL APIENTRY DllMain(HMODULE /* hModule */, DWORD ul_reason_for_call, - LPVOID /* lpReserved */ -) { - switch (ul_reason_for_call) { - case DLL_PROCESS_ATTACH: - case DLL_THREAD_ATTACH: - case DLL_THREAD_DETACH: - case DLL_PROCESS_DETACH: - break; - } - return TRUE; -} diff --git a/cdc_rsync/error_messages.h b/cdc_rsync/error_messages.h deleted file mode 100644 index 6268a23..0000000 --- a/cdc_rsync/error_messages.h +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef CDC_RSYNC_ERROR_MESSAGES_H_ -#define CDC_RSYNC_ERROR_MESSAGES_H_ - -namespace cdc_ft { - -// Server connection timed out. SSH probably stale. -constexpr char kMsgFmtConnectionTimeout[] = - "Server connection timed out. Please re-run 'ggp ssh init' and verify that " - "the IP '%s' and the port '%i' are correct."; - -// Server connection timed out and IP was not passed in. Probably network error. -constexpr char kMsgConnectionTimeoutWithIp[] = - "Server connection timed out. Please check your network connection."; - -// Receiving pipe end was shut down unexpectedly. -constexpr char kMsgConnectionLost[] = - "The connection to the instance was shut down unexpectedly."; - -// Binding to the port failed. -constexpr char kMsgAddressInUse[] = - "Failed to establish a connection to the instance. All ports are already " - "in use. This can happen if another instance of this command is running. " - "Currently, only 10 simultaneous connections are supported."; - -// Deployment failed even though gamelet components were copied successfully. -constexpr char kMsgDeployFailed[] = - "Failed to deploy the instance components for unknown reasons. " - "Please report this issue."; - -// Picking an instance is not allowed in quiet mode. -constexpr char kMsgInstancePickerNotAvailableInQuietMode[] = - "Multiple gamelet instances are reserved, but the instance picker is not " - "available in quiet mode. Please specify --instance or remove -q resp. " - "--quiet."; - -} // namespace cdc_ft - -#endif // CDC_RSYNC_ERROR_MESSAGES_H_ diff --git a/cdc_rsync/main.cc b/cdc_rsync/main.cc new file mode 100644 index 0000000..3dd475e --- /dev/null +++ b/cdc_rsync/main.cc @@ -0,0 +1,147 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#define WIN32_LEAN_AND_MEAN +#include + +#include +#include + +#include "cdc_rsync/cdc_rsync_client.h" +#include "cdc_rsync/params.h" +#include "common/log.h" +#include "common/status.h" +#include "common/util.h" + +namespace { + +enum class ReturnCode { + // No error. Will match the tool's exit code, so OK must be 0. + kOk = 0, + + // Generic error. + kGenericError = 1, + + // Server connection timed out. + kConnectionTimeout = 2, + + // Connection to the server was shut down unexpectedly. + kConnectionLost = 3, + + // Binding to the forward port failed, probably because there's another + // instance of cdc_rsync running. + kAddressInUse = 4, + + // Server deployment failed. This should be rare, it means that the server + // components were successfully copied, but the up-to-date check still fails. + kDeployFailed = 5, +}; + +ReturnCode TagToMessage(cdc_ft::Tag tag, + const cdc_ft::params::Parameters& params, + std::string* msg) { + msg->clear(); + switch (tag) { + case cdc_ft::Tag::kSocketEof: + // Receiving pipe end was shut down unexpectedly. + *msg = "The connection to the instance was shut down unexpectedly."; + return ReturnCode::kConnectionLost; + + case cdc_ft::Tag::kAddressInUse: + *msg = + "Failed to establish a connection to the instance. All ports are " + "already in use. This can happen if another instance of this command " + "is running. Currently, only 10 simultaneous connections are " + "supported."; + return ReturnCode::kAddressInUse; + + case cdc_ft::Tag::kDeployServer: + *msg = + "Failed to deploy the instance components for unknown reasons. " + "Please report this issue."; + return ReturnCode::kDeployFailed; + + case cdc_ft::Tag::kConnectionTimeout: + // Server connection timed out. SSH probably stale. + *msg = absl::StrFormat( + "Server connection timed out. Verify that host '%s' and port '%i' " + "are correct, or specify a larger timeout with --contimeout.", + params.user_host, params.options.port); + return ReturnCode::kConnectionTimeout; + + case cdc_ft::Tag::kCount: + return ReturnCode::kGenericError; + } + + // Should not happen (TM). Will fall back to status message in this case. + return ReturnCode::kGenericError; +} + +} // namespace + +int wmain(int argc, wchar_t* argv[]) { + // Convert args from wide to UTF8 strings. + std::vector utf8_str_args; + utf8_str_args.reserve(argc); + for (int i = 0; i < argc; i++) { + utf8_str_args.push_back(cdc_ft::Util::WideToUtf8Str(argv[i])); + } + + // Convert args from UTF8 strings to UTF8 c-strings. + std::vector utf8_args; + utf8_args.reserve(argc); + for (const auto& utf8_str_arg : utf8_str_args) { + utf8_args.push_back(utf8_str_arg.c_str()); + } + + // Read parameters from the environment and the command line. + cdc_ft::params::Parameters parameters; + if (!cdc_ft::params::Parse(argc, utf8_args.data(), ¶meters)) { + return 1; + } + + // Initialize logging. + cdc_ft::LogLevel log_level = + cdc_ft::Log::VerbosityToLogLevel(parameters.options.verbosity); + cdc_ft::Log::Initialize(std::make_unique(log_level)); + + // Run rsync. + cdc_ft::CdcRsyncClient client(parameters.options, parameters.sources, + parameters.user_host, parameters.destination); + absl::Status status = client.Run(); + if (status.ok()) { + return static_cast(ReturnCode::kOk); + } + + // Get an error message from the tag associated with the status. + std::string error_message; + ReturnCode code = ReturnCode::kGenericError; + absl::optional tag = cdc_ft::GetTag(status); + if (tag.has_value()) { + code = TagToMessage(tag.value(), parameters, &error_message); + } + + // Fall back to status message if there was no tag. + if (error_message.empty()) { + error_message = status.message(); + } else if (parameters.options.verbosity >= 2) { + // In verbose mode, log the status as well, so nothing gets lost. + LOG_ERROR("%s", status.ToString()); + } + + if (!error_message.empty()) { + fprintf(stderr, "Error: %s\n", error_message.c_str()); + } + return static_cast(code); +} diff --git a/cdc_rsync_cli/params.cc b/cdc_rsync/params.cc similarity index 74% rename from cdc_rsync_cli/params.cc rename to cdc_rsync/params.cc index 665b14a..d98597a 100644 --- a/cdc_rsync_cli/params.cc +++ b/cdc_rsync/params.cc @@ -12,12 +12,13 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "cdc_rsync_cli/params.h" +#include "cdc_rsync/params.h" #include #include "absl/status/status.h" #include "absl/strings/str_format.h" +#include "absl/strings/str_split.h" #include "common/path.h" #include "lib/zstd.h" @@ -25,6 +26,8 @@ namespace cdc_ft { namespace params { namespace { +using Options = CdcRsyncClient::Options; + template void PrintError(const absl::FormatSpec& format, Args... args) { std::cerr << "Error: " << absl::StrFormat(format, args...) << std::endl; @@ -39,11 +42,13 @@ Synchronizes local files and files on a gamelet. Matching files are skipped. For partially matching files only the deltas are transferred. Usage: - cdc_rsync [options] source [source]... destination + cdc_rsync [options] source [source]... [user@]host:destination Parameters: - source Local file or folder to be copied - destination Destination folder on the gamelet + source Local file or directory to be copied + user Remote SSH user name + host Remote host or IP address + destination Remote destination directory Options: --ip string Gamelet IP. Required. @@ -54,7 +59,7 @@ Options: --json Print JSON progress -n, --dry-run Perform a trial run with no changes made -r, --recursive Recurse into directories - --delete Delete extraneous files from destination folder + --delete Delete extraneous files from destination directory -z, --compress Compress file data during the transfer --compress-level num Explicitly set compression level (default: 6) -c, --checksum Skip files based on checksum, not mod-time & size @@ -68,13 +73,29 @@ Options: -R, --relative Use relative path names --existing Skip creating new files on instance --copy-dest dir Use files from dir as sync base if files are missing - from destination folder + --ssh-command Path and arguments of SSH command to use, e.g. + C:\path\to\ssh.exe -F config -i id_rsa -oStrictHostKeyChecking=yes -oUserKnownHostsFile="""known_hosts""" + Can also be specified by the CDC_SSH_COMMAND environment variable. + --scp-command Path and arguments of SSH command to use, e.g. + C:\path\to\scp.exe -F config -i id_rsa -oStrictHostKeyChecking=yes -oUserKnownHostsFile="""known_hosts""" + Can also be specified by the CDC_SCP_COMMAND environment variable. -h --help Help for cdc_rsync )"; +constexpr char kSshCommandEnvVar[] = "CDC_SSH_COMMAND"; +constexpr char kScpCommandEnvVar[] = "CDC_SCP_COMMAND"; + +// Populates some parameters from environment variables. +void PopulateFromEnvVars(Parameters* parameters) { + path::GetEnv(kSshCommandEnvVar, ¶meters->options.ssh_command) + .IgnoreError(); + path::GetEnv(kScpCommandEnvVar, ¶meters->options.scp_command) + .IgnoreError(); +} + // Handles the --exclude-from and --include-from options. OptionResult HandleFilterRuleFile(const std::string& option_name, - const char* path, FilterRule::Type type, + const char* path, PathFilter::Rule::Type type, Parameters* params) { if (!path) { PrintError("Option '%s' needs a value", option_name); @@ -92,7 +113,7 @@ OptionResult HandleFilterRuleFile(const std::string& option_name, } for (std::string& pattern : patterns) { - params->filter_rules.emplace_back(type, std::move(pattern)); + params->options.filter.AddRule(type, std::move(pattern)); } return OptionResult::kConsumedKeyValue; } @@ -143,11 +164,6 @@ bool LoadFilesFrom(const std::string& files_from, OptionResult HandleParameter(const std::string& key, const char* value, Parameters* params, bool* help) { - if (key == "ip") { - params->options.ip = value; - return OptionResult::kConsumedKeyValue; - } - if (key == "port") { if (value) { params->options.port = atoi(value); @@ -181,27 +197,29 @@ OptionResult HandleParameter(const std::string& key, const char* value, } if (key == "include") { - params->filter_rules.emplace_back(FilterRule::Type::kInclude, value); + params->options.filter.AddRule(PathFilter::Rule::Type::kInclude, value); return OptionResult::kConsumedKeyValue; } if (key == "include-from") { - return HandleFilterRuleFile(key, value, FilterRule::Type::kInclude, params); + return HandleFilterRuleFile(key, value, PathFilter::Rule::Type::kInclude, + params); } if (key == "exclude") { - params->filter_rules.emplace_back(FilterRule::Type::kExclude, value); + params->options.filter.AddRule(PathFilter::Rule::Type::kExclude, value); return OptionResult::kConsumedKeyValue; } if (key == "exclude-from") { - return HandleFilterRuleFile(key, value, FilterRule::Type::kExclude, params); + return HandleFilterRuleFile(key, value, PathFilter::Rule::Type::kExclude, + params); } if (key == "files-from") { // Implies -R. params->options.relative = true; - params->files_from = value; + params->files_from = value ? value : std::string(); return OptionResult::kConsumedKeyValue; } @@ -250,7 +268,7 @@ OptionResult HandleParameter(const std::string& key, const char* value, } if (key == "copy-dest") { - params->options.copy_dest = value; + params->options.copy_dest = value ? value : std::string(); return OptionResult::kConsumedKeyValue; } @@ -259,13 +277,23 @@ OptionResult HandleParameter(const std::string& key, const char* value, return OptionResult::kConsumedKey; } + if (key == "ssh-command") { + params->options.ssh_command = value ? value : std::string(); + return OptionResult::kConsumedKeyValue; + } + + if (key == "scp-command") { + params->options.scp_command = value ? value : std::string(); + return OptionResult::kConsumedKeyValue; + } + PrintError("Unknown option: '%s'", key); return OptionResult::kError; } -bool CheckParameters(const Parameters& params, bool help) { +bool ValidateParameters(const Parameters& params, bool help) { if (help) { - printf("%s", kHelpText); + std::cout << kHelpText; return false; } @@ -274,13 +302,7 @@ bool CheckParameters(const Parameters& params, bool help) { return false; } - if (!params.options.ip || params.options.ip[0] == '\0') { - PrintError("--ip must specify a valid IP address"); - return false; - } - - if (!params.options.port || params.options.port <= 0 || - params.options.port > UINT16_MAX) { + if (params.options.port <= 0 || params.options.port > UINT16_MAX) { PrintError("--port must specify a valid port"); return false; } @@ -301,9 +323,10 @@ bool CheckParameters(const Parameters& params, bool help) { // Warn that any include rules not followed by an exclude rule are pointless // as the files would be included, anyway. - for (int n = static_cast(params.filter_rules.size()) - 1; n >= 0; --n) { - const Parameters::FilterRule& rule = params.filter_rules[n]; - if (rule.type == FilterRule::Type::kExclude) { + const std::vector& rules = params.options.filter.GetRules(); + for (int n = static_cast(rules.size()) - 1; n >= 0; --n) { + const PathFilter::Rule& rule = rules[n]; + if (rule.type == PathFilter::Rule::Type::kExclude) { break; } std::cout << "Warning: Include pattern '" << rule.pattern @@ -311,6 +334,31 @@ bool CheckParameters(const Parameters& params, bool help) { << std::endl; } + if (params.sources.empty() && params.destination.empty()) { + PrintError("Missing source and destination"); + return false; + } + + if (params.destination.empty()) { + PrintError("Missing destination"); + return false; + } + + if (params.sources.empty()) { + // If one arg was passed on the command line, it is not clear whether it + // was supposed to be a source or destination. + PrintError("Missing source or destination"); + return false; + } + + if (params.user_host.empty()) { + PrintError( + "No remote host specified in destination '%s'. " + "Expected [user@]host:destination.", + params.destination); + return false; + } + return true; } @@ -335,6 +383,25 @@ bool CheckOptionResult(OptionResult result, const std::string& name, return true; } +// Removes the user/host part of |destination| and puts it into |user_host|, +// e.g. if |destination| is initially "user@foo.com:~/file", it is "~/file" +// afterward and |user_host| is |user@foo.com|. Does not touch Windows drives, +// e.g. C:\foo. +void PopUserHost(std::string* destination, std::string* user_host) { + std::vector parts = + absl::StrSplit(*destination, absl::MaxSplits(':', 1)); + if (parts.size() < 2) return; + + // Don't mistake the C part of C:\foo as user/host. + if (parts[0].size() == 1 && toupper(parts[0][0]) >= 'A' && + toupper(parts[0][0]) <= 'Z') { + return; + } + + *user_host = parts[0]; + *destination = parts[1]; +} + } // namespace const char* HelpText() { return kHelpText; } @@ -349,6 +416,9 @@ bool Parse(int argc, const char* const* argv, Parameters* parameters) { return false; } + // Before applying args, populate parameters from env vars. + PopulateFromEnvVars(parameters); + bool help = false; for (int index = 1; index < argc; ++index) { // Handle '--key [value]' and '--key=value' options. @@ -404,34 +474,15 @@ bool Parse(int argc, const char* const* argv, Parameters* parameters) { // Load files-from file (can't do it when --files-from is handled since not // all sources might have been read at that point. - if (parameters->files_from && + if (!parameters->files_from.empty() && !LoadFilesFrom(parameters->files_from, ¶meters->sources, - ¶meters->sources_dir)) { + ¶meters->options.sources_dir)) { return false; } - if (!CheckParameters(*parameters, help)) { - return false; - } + PopUserHost(¶meters->destination, ¶meters->user_host); - if (parameters->sources.empty() && parameters->destination.empty()) { - PrintError("Missing source and destination"); - return false; - } - - if (parameters->destination.empty()) { - PrintError("Missing destination"); - return false; - } - - if (parameters->sources.empty()) { - // If one arg was passed on the command line, it is not clear whether it - // was supposed to be a source or destination. Try to infer that, e.g. - // cdc_rsync *.txt -> Missing destination - // cdc_rsync /mnt/developer -> Missing source - bool missing_src = parameters->destination[0] == '/'; - - PrintError("Missing %s", missing_src ? "source" : "destination"); + if (!ValidateParameters(*parameters, help)) { return false; } diff --git a/cdc_rsync_cli/params.h b/cdc_rsync/params.h similarity index 64% rename from cdc_rsync_cli/params.h rename to cdc_rsync/params.h index 97f0ec0..e278a21 100644 --- a/cdc_rsync_cli/params.h +++ b/cdc_rsync/params.h @@ -14,34 +14,24 @@ * limitations under the License. */ -#ifndef CDC_RSYNC_CLI_PARAMS_H_ -#define CDC_RSYNC_CLI_PARAMS_H_ +#ifndef CDC_RSYNC_PARAMS_H_ +#define CDC_RSYNC_PARAMS_H_ #include #include -#include "cdc_rsync/cdc_rsync.h" +#include "cdc_rsync/cdc_rsync_client.h" namespace cdc_ft { namespace params { // All cdc_rsync command line parameters. struct Parameters { - // Copy of cdc_ft::FilterRule with std::string instead of const char*. - struct FilterRule { - using Type = ::cdc_ft::FilterRule::Type; - FilterRule(Type type, std::string pattern) - : type(type), pattern(std::move(pattern)) {} - Type type; - std::string pattern; - }; - - Options options; - std::vector filter_rules; + CdcRsyncClient::Options options; std::vector sources; + std::string user_host; std::string destination; - const char* files_from = nullptr; - std::string sources_dir; // Base directory for files loaded for --files-from. + std::string files_from; }; // Parses sources, destination and options from the command line args. @@ -51,4 +41,4 @@ bool Parse(int argc, const char* const* argv, Parameters* parameters); } // namespace params } // namespace cdc_ft -#endif // CDC_RSYNC_CLI_PARAMS_H_ +#endif // CDC_RSYNC_PARAMS_H_ diff --git a/cdc_rsync_cli/params_test.cc b/cdc_rsync/params_test.cc similarity index 55% rename from cdc_rsync_cli/params_test.cc rename to cdc_rsync/params_test.cc index 1315427..ea3b8c0 100644 --- a/cdc_rsync_cli/params_test.cc +++ b/cdc_rsync/params_test.cc @@ -12,11 +12,12 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "cdc_rsync_cli/params.h" +#include "cdc_rsync/params.h" #include "absl/strings/match.h" #include "common/log.h" #include "common/path.h" +#include "common/status_test_macros.h" #include "common/test_main.h" #include "gtest/gtest.h" @@ -24,6 +25,13 @@ namespace cdc_ft { namespace params { namespace { +using Options = CdcRsyncClient::Options; + +constexpr char kSrc[] = "source"; +constexpr char kUserHostDst[] = "user@host:destination"; +constexpr char kUserHost[] = "user@host"; +constexpr char kDst[] = "destination"; + class TestLog : public Log { public: explicit TestLog() : Log(LogLevel::kInfo) {} @@ -44,9 +52,15 @@ std::string NeedsValueError(const char* option_name) { class ParamsTest : public ::testing::Test { public: - void SetUp() override { prev_stderr_ = std::cerr.rdbuf(errors_.rdbuf()); } + void SetUp() override { + prev_stdout_ = std::cout.rdbuf(output_.rdbuf()); + prev_stderr_ = std::cerr.rdbuf(errors_.rdbuf()); + } - void TearDown() override { std::cerr.rdbuf(prev_stderr_); } + void TearDown() override { + std::cout.rdbuf(prev_stdout_); + std::cerr.rdbuf(prev_stderr_); + } protected: void ExpectNoError() const { @@ -54,6 +68,12 @@ class ParamsTest : public ::testing::Test { << "Expected empty stderr but got\n'" << errors_.str() << "'"; } + void ExpectOutput(const std::string& expected) const { + EXPECT_TRUE(absl::StrContains(output_.str(), expected)) + << "Expected stdout to contain '" << expected << "' but got\n'" + << output_.str() << "'"; + } + void ExpectError(const std::string& expected) const { EXPECT_TRUE(absl::StrContains(errors_.str(), expected)) << "Expected stderr to contain '" << expected << "' but got\n'" @@ -68,16 +88,16 @@ class ParamsTest : public ::testing::Test { path::Join(base_dir_, "empty_source_files.txt"); Parameters parameters_; + std::stringstream output_; std::stringstream errors_; + std::streambuf* prev_stdout_; std::streambuf* prev_stderr_; }; TEST_F(ParamsTest, ParseSucceedsDefaults) { - const char* argv[] = {"cdc_rsync.exe", "--ip=1.2.3.4", "--port=1234", - "source", "destination", NULL}; + const char* argv[] = {"cdc_rsync.exe", kSrc, kUserHostDst, NULL}; EXPECT_TRUE(Parse(static_cast(std::size(argv)) - 1, argv, ¶meters_)); - EXPECT_STREQ("1.2.3.4", parameters_.options.ip); - EXPECT_EQ(1234, parameters_.options.port); + EXPECT_EQ(RemoteUtil::kDefaultSshPort, parameters_.options.port); EXPECT_FALSE(parameters_.options.delete_); EXPECT_FALSE(parameters_.options.recursive); EXPECT_EQ(0, parameters_.options.verbosity); @@ -86,19 +106,19 @@ TEST_F(ParamsTest, ParseSucceedsDefaults) { EXPECT_FALSE(parameters_.options.compress); EXPECT_FALSE(parameters_.options.checksum); EXPECT_FALSE(parameters_.options.dry_run); - EXPECT_EQ(parameters_.options.copy_dest, nullptr); + EXPECT_TRUE(parameters_.options.copy_dest.empty()); EXPECT_EQ(6, parameters_.options.compress_level); EXPECT_EQ(10, parameters_.options.connection_timeout_sec); EXPECT_EQ(1, parameters_.sources.size()); - EXPECT_EQ(parameters_.sources[0], "source"); - EXPECT_EQ(parameters_.destination, "destination"); + EXPECT_EQ(parameters_.sources[0], kSrc); + EXPECT_EQ(parameters_.user_host, kUserHost); + EXPECT_EQ(parameters_.destination, kDst); ExpectNoError(); } TEST_F(ParamsTest, ParseSucceedsWithOptionFromTwoArguments) { const char* argv[] = { - "cdc_rsync.exe", "--ip=1.2.3.4", "--port=1234", "--compress-level", "2", - "source", "destination", NULL}; + "cdc_rsync.exe", "--compress-level", "2", kSrc, kUserHostDst, NULL}; EXPECT_TRUE(Parse(static_cast(std::size(argv)) - 1, argv, ¶meters_)); EXPECT_EQ(parameters_.options.compress_level, 2); ExpectNoError(); @@ -106,68 +126,104 @@ TEST_F(ParamsTest, ParseSucceedsWithOptionFromTwoArguments) { TEST_F(ParamsTest, ParseSucceedsWithOptionFromOneArgumentWithEqualityWithValue) { - const char* argv[] = { - "cdc_rsync.exe", "--ip=1.2.3.4", "--port=1234", "--compress-level=2", - "source", "destination", NULL}; + const char* argv[] = {"cdc_rsync.exe", "--compress-level=2", kSrc, + kUserHostDst, NULL}; EXPECT_TRUE(Parse(static_cast(std::size(argv)) - 1, argv, ¶meters_)); ASSERT_EQ(parameters_.sources.size(), 1); EXPECT_EQ(parameters_.options.compress_level, 2); - EXPECT_EQ(parameters_.sources[0], "source"); - EXPECT_EQ(parameters_.destination, "destination"); + EXPECT_EQ(parameters_.sources[0], kSrc); + EXPECT_EQ(parameters_.user_host, kUserHost); + EXPECT_EQ(parameters_.destination, kDst); ExpectNoError(); } TEST_F(ParamsTest, ParseFailsOnCompressLevelEqualsNoValue) { - const char* argv[] = {"cdc_rsync.exe", "--compress-level=", "source", - "destination", NULL}; + const char* argv[] = {"cdc_rsync.exe", "--compress-level=", kSrc, + kUserHostDst, NULL}; EXPECT_FALSE( Parse(static_cast(std::size(argv)) - 1, argv, ¶meters_)); ExpectError(NeedsValueError("compress-level")); } TEST_F(ParamsTest, ParseFailsOnPortEqualsNoValue) { - const char* argv[] = {"cdc_rsync.exe", "--port=", "source", "destination", - NULL}; + const char* argv[] = {"cdc_rsync.exe", "--port=", kSrc, kUserHostDst, NULL}; EXPECT_FALSE( Parse(static_cast(std::size(argv)) - 1, argv, ¶meters_)); ExpectError(NeedsValueError("port")); } TEST_F(ParamsTest, ParseFailsOnContimeoutEqualsNoValue) { - const char* argv[] = {"cdc_rsync.exe", "--contimeout=", "source", - "destination", NULL}; + const char* argv[] = {"cdc_rsync.exe", "--contimeout=", kSrc, kUserHostDst, + NULL}; EXPECT_FALSE( Parse(static_cast(std::size(argv)) - 1, argv, ¶meters_)); ExpectError(NeedsValueError("contimeout")); } -TEST_F(ParamsTest, ParseFailsOnIpEqualsNoValue) { - const char* argv[] = {"cdc_rsync.exe", "--ip=", "source", "destination", +TEST_F(ParamsTest, ParseSucceedsWithSshScpCommands) { + const char* argv[] = {"cdc_rsync.exe", kSrc, + kUserHostDst, "--ssh-command=sshcmd", + "--scp-command=scpcmd", NULL}; + EXPECT_TRUE(Parse(static_cast(std::size(argv)) - 1, argv, ¶meters_)); + EXPECT_EQ(parameters_.options.scp_command, "scpcmd"); + EXPECT_EQ(parameters_.options.ssh_command, "sshcmd"); +} + +TEST_F(ParamsTest, ParseSucceedsWithSshScpCommandsByEnvVars) { + EXPECT_OK(path::SetEnv("CDC_SSH_COMMAND", "sshcmd")); + EXPECT_OK(path::SetEnv("CDC_SCP_COMMAND", "scpcmd")); + const char* argv[] = {"cdc_rsync.exe", kSrc, kUserHostDst, NULL}; + EXPECT_TRUE(Parse(static_cast(std::size(argv)) - 1, argv, ¶meters_)); + EXPECT_EQ(parameters_.options.scp_command, "scpcmd"); + EXPECT_EQ(parameters_.options.ssh_command, "sshcmd"); +} + +TEST_F(ParamsTest, ParseSucceedsWithNoSshCommand) { + const char* argv[] = {"cdc_rsync.exe", kSrc, kUserHostDst, + "--ssh-command=", NULL}; + EXPECT_FALSE( + Parse(static_cast(std::size(argv)) - 1, argv, ¶meters_)); + ExpectError(NeedsValueError("ssh-command")); +} + +TEST_F(ParamsTest, ParseSucceedsWithNoScpCommand) { + const char* argv[] = {"cdc_rsync.exe", kSrc, kUserHostDst, "--scp-command", NULL}; EXPECT_FALSE( Parse(static_cast(std::size(argv)) - 1, argv, ¶meters_)); - ExpectError(NeedsValueError("ip")); + ExpectError(NeedsValueError("scp-command")); +} + +TEST_F(ParamsTest, ParseFailsOnNoUserHost) { + const char* argv[] = {"cdc_rsync.exe", kSrc, kDst, NULL}; + EXPECT_FALSE( + Parse(static_cast(std::size(argv)) - 1, argv, ¶meters_)); + ExpectError("No remote host specified"); +} + +TEST_F(ParamsTest, ParseDoesNotThinkCIsAHost) { + const char* argv[] = {"cdc_rsync.exe", kSrc, "C:\\foo", NULL}; + EXPECT_FALSE( + Parse(static_cast(std::size(argv)) - 1, argv, ¶meters_)); + ExpectError("No remote host specified"); } TEST_F(ParamsTest, ParseWithoutParametersFailsOnMissingSourceAndDestination) { - const char* argv[] = {"cdc_rsync.exe", "--ip=1.2.3.4", "--port=1234", NULL}; + const char* argv[] = {"cdc_rsync.exe", NULL}; EXPECT_FALSE( Parse(static_cast(std::size(argv)) - 1, argv, ¶meters_)); - ExpectError("Missing source"); + ExpectOutput("Usage:"); } TEST_F(ParamsTest, ParseWithSingleParameterFailsOnMissingDestination) { - const char* argv[] = {"cdc_rsync.exe", "--ip=1.2.3.4", "--port=1234", - "source", NULL}; + const char* argv[] = {"cdc_rsync.exe", kSrc, NULL}; EXPECT_FALSE( Parse(static_cast(std::size(argv)) - 1, argv, ¶meters_)); - ExpectError("Missing destination"); + ExpectError("Missing source or destination"); } -TEST_F(ParamsTest, ParseSuccessedsWithMultipleLetterKeyConsumed) { - const char* argv[] = { - "cdc_rsync.exe", "--ip=1.2.3.4", "--port=1234", "-rvqWRzcn", - "source", "destination", NULL}; +TEST_F(ParamsTest, ParseSucceedsWithMultipleLetterKeyConsumed) { + const char* argv[] = {"cdc_rsync.exe", "-rvqWRzcn", kSrc, kUserHostDst, NULL}; EXPECT_TRUE(Parse(static_cast(std::size(argv)) - 1, argv, ¶meters_)); EXPECT_TRUE(parameters_.options.recursive); EXPECT_EQ(parameters_.options.verbosity, 1); @@ -182,17 +238,15 @@ TEST_F(ParamsTest, ParseSuccessedsWithMultipleLetterKeyConsumed) { TEST_F(ParamsTest, ParseFailsOnMultipleLetterKeyConsumedOptionsWithUnsupportedOne) { - const char* argv[] = {"cdc_rsync.exe", "-rvqaWRzcn", "source", "destination", + const char* argv[] = {"cdc_rsync.exe", "-rvqaWRzcn", kSrc, kUserHostDst, NULL}; EXPECT_FALSE( Parse(static_cast(std::size(argv)) - 1, argv, ¶meters_)); ExpectError("Unknown option: 'a'"); } -TEST_F(ParamsTest, ParseSuccessedsWithMultipleLongKeyConsumedOptions) { +TEST_F(ParamsTest, ParseSucceedsWithMultipleLongKeyConsumedOptions) { const char* argv[] = {"cdc_rsync.exe", - "--ip=1.2.3.4", - "--port=1234", "--recursive", "--verbosity", "--quiet", @@ -204,8 +258,8 @@ TEST_F(ParamsTest, ParseSuccessedsWithMultipleLongKeyConsumedOptions) { "--dry-run", "--existing", "--json", - "source", - "destination", + kSrc, + kUserHostDst, NULL}; EXPECT_TRUE(Parse(static_cast(std::size(argv)) - 1, argv, ¶meters_)); EXPECT_TRUE(parameters_.options.recursive); @@ -223,51 +277,42 @@ TEST_F(ParamsTest, ParseSuccessedsWithMultipleLongKeyConsumedOptions) { } TEST_F(ParamsTest, ParseFailsOnUnknownKey) { - const char* argv[] = {"cdc_rsync.exe", "-unknownKey", "source", "destination", + const char* argv[] = {"cdc_rsync.exe", "-unknownKey", kSrc, kUserHostDst, NULL}; EXPECT_FALSE( Parse(static_cast(std::size(argv)) - 1, argv, ¶meters_)); ExpectError("Unknown option: 'u'"); } -TEST_F(ParamsTest, ParseSuccessedsWithSupportedKeyValue) { +TEST_F(ParamsTest, ParseSucceedsWithSupportedKeyValue) { const char* argv[] = { - "cdc_rsync.exe", "--compress-level", "11", "--port=4086", - "--ip=127.0.0.1", "--contimeout", "99", "--copy-dest=dest", - "source", "destination", NULL}; + "cdc_rsync.exe", "--compress-level", "11", "--contimeout", "99", "--port", + "4086", "--copy-dest=dest", kSrc, kUserHostDst, NULL}; EXPECT_TRUE(Parse(static_cast(std::size(argv)) - 1, argv, ¶meters_)); EXPECT_EQ(parameters_.options.compress_level, 11); EXPECT_EQ(parameters_.options.connection_timeout_sec, 99); EXPECT_EQ(parameters_.options.port, 4086); - EXPECT_STREQ(parameters_.options.ip, "127.0.0.1"); - EXPECT_STREQ(parameters_.options.copy_dest, "dest"); + EXPECT_EQ(parameters_.options.copy_dest, "dest"); ExpectNoError(); } -TEST_F(ParamsTest, - ParseSuccessedsWithSupportedKeyValueWithoutEqualityForChars) { - const char* argv[] = {"cdc_rsync.exe", "--port", "4086", "--ip", - "127.0.0.1", "--copy-dest", "dest", "source", - "destination", NULL}; +TEST_F(ParamsTest, ParseSucceedsWithSupportedKeyValueWithoutEqualityForChars) { + const char* argv[] = {"cdc_rsync.exe", "--copy-dest", "dest", kSrc, + kUserHostDst, NULL}; EXPECT_TRUE(Parse(static_cast(std::size(argv)) - 1, argv, ¶meters_)); - EXPECT_EQ(parameters_.options.port, 4086); - EXPECT_STREQ(parameters_.options.ip, "127.0.0.1"); - EXPECT_STREQ(parameters_.options.copy_dest, "dest"); + EXPECT_EQ(parameters_.options.copy_dest, "dest"); ExpectNoError(); } -TEST_F(ParamsTest, ParseFailsOnGameletIpNeedsPort) { - const char* argv[] = {"cdc_rsync.exe", "--ip=127.0.0.1", "source", - "destination", NULL}; +TEST_F(ParamsTest, ParseFailsOnInvalidPort) { + const char* argv[] = {"cdc_rsync.exe", "--port=0", kSrc, kUserHostDst, NULL}; EXPECT_FALSE( Parse(static_cast(std::size(argv)) - 1, argv, ¶meters_)); ExpectError("--port must specify a valid port"); } TEST_F(ParamsTest, ParseFailsOnDeleteNeedsRecursive) { - const char* argv[] = { - "cdc_rsync.exe", "--ip=1.2.3.4", "--port=1234", "--delete", - "source", "destination", NULL}; + const char* argv[] = {"cdc_rsync.exe", "--delete", kSrc, kUserHostDst, NULL}; EXPECT_FALSE( Parse(static_cast(std::size(argv)) - 1, argv, ¶meters_)); ExpectError("--delete does not work without --recursive (-r)"); @@ -281,10 +326,10 @@ TEST_F(ParamsTest, ParseChecksCompressLevel) { for (int n = 0; n < std::size(levels); ++n) { std::string level = "--compress-level=" + std::to_string(levels[n]); - const char* argv[] = {"cdc_rsync.exe", "--ip=1.2.3.4", "--port=1234", - level.c_str(), "source", "destination"}; - EXPECT_TRUE(Parse(static_cast(std::size(argv)) - 1, argv, - ¶meters_) == valid[n]); + const char* argv[] = {"cdc_rsync.exe", level.c_str(), kSrc, kUserHostDst, + NULL}; + EXPECT_EQ(Parse(static_cast(std::size(argv)) - 1, argv, ¶meters_), + valid[n]); if (valid[n]) { ExpectNoError(); } else { @@ -295,94 +340,95 @@ TEST_F(ParamsTest, ParseChecksCompressLevel) { } TEST_F(ParamsTest, ParseFailsOnUnknownKeyValue) { - const char* argv[] = {"cdc_rsync.exe", "--unknownKey=5", "source", - "destination", NULL}; + const char* argv[] = {"cdc_rsync.exe", "--unknownKey=5", kSrc, kUserHostDst, + NULL}; EXPECT_FALSE( Parse(static_cast(std::size(argv)) - 1, argv, ¶meters_)); ExpectError("unknownKey"); } TEST_F(ParamsTest, ParseFailsWithHelpOption) { - const char* argv[] = {"cdc_rsync.exe", "--ip=1.2.3.4", "--port=1234", - "source", "destination", NULL}; + const char* argv[] = {"cdc_rsync.exe", kSrc, kUserHostDst, NULL}; EXPECT_TRUE(Parse(static_cast(std::size(argv)) - 1, argv, ¶meters_)); - const char* argv2[] = { - "cdc_rsync.exe", "--ip=1.2.3.4", "--port=1234", "source", - "destination", "--help", NULL}; + const char* argv2[] = {"cdc_rsync.exe", kSrc, kUserHostDst, "--help", NULL}; EXPECT_FALSE( Parse(static_cast(std::size(argv2)) - 1, argv2, ¶meters_)); ExpectNoError(); - const char* argv3[] = { - "cdc_rsync.exe", "--ip=1.2.3.4", "--port=1234", "source", - "destination", "-h", NULL}; + const char* argv3[] = {"cdc_rsync.exe", kSrc, kUserHostDst, "-h", NULL}; EXPECT_FALSE( Parse(static_cast(std::size(argv3)) - 1, argv3, ¶meters_)); ExpectNoError(); } TEST_F(ParamsTest, ParseSucceedsWithIncludeExclude) { - const char* argv[] = { - "cdc_rsync.exe", "--ip=1.2.3.4", "--port=1234", "--include=*.txt", - "--exclude", "*.dat", "--include", "*.exe", - "source", "destination", NULL}; + const char* argv[] = {"cdc_rsync.exe", + "--include=*.txt", + "--exclude", + "*.dat", + "--include", + "*.exe", + kSrc, + kUserHostDst, + NULL}; EXPECT_TRUE(Parse(static_cast(std::size(argv)) - 1, argv, ¶meters_)); - ASSERT_EQ(parameters_.filter_rules.size(), 3); - ASSERT_EQ(parameters_.filter_rules[0].type, FilterRule::Type::kInclude); - ASSERT_EQ(parameters_.filter_rules[0].pattern, "*.txt"); - ASSERT_EQ(parameters_.filter_rules[1].type, FilterRule::Type::kExclude); - ASSERT_EQ(parameters_.filter_rules[1].pattern, "*.dat"); - ASSERT_EQ(parameters_.filter_rules[2].type, FilterRule::Type::kInclude); - ASSERT_EQ(parameters_.filter_rules[2].pattern, "*.exe"); + const std::vector& rules = + parameters_.options.filter.GetRules(); + ASSERT_EQ(rules.size(), 3); + ASSERT_EQ(rules[0].type, PathFilter::Rule::Type::kInclude); + ASSERT_EQ(rules[0].pattern, "*.txt"); + ASSERT_EQ(rules[1].type, PathFilter::Rule::Type::kExclude); + ASSERT_EQ(rules[1].pattern, "*.dat"); + ASSERT_EQ(rules[2].type, PathFilter::Rule::Type::kInclude); + ASSERT_EQ(rules[2].pattern, "*.exe"); ExpectNoError(); } TEST_F(ParamsTest, FilesFrom_NoFile) { - const char* argv[] = { - "cdc_rsync.exe", "--ip=1.2.3.4", "--port=1234", "source", - "destination", "--files-from", NULL}; + const char* argv[] = {"cdc_rsync.exe", kSrc, kUserHostDst, "--files-from", + NULL}; EXPECT_FALSE( Parse(static_cast(std::size(argv)) - 1, argv, ¶meters_)); ExpectError(NeedsValueError("files-from")); } TEST_F(ParamsTest, FilesFrom_ImpliesRelative) { - const char* argv[] = { - "cdc_rsync.exe", "--ip=1.2.3.4", "--port=1234", "--files-from", - sources_file_.c_str(), base_dir_.c_str(), "destination", NULL}; + const char* argv[] = {"cdc_rsync.exe", "--files-from", + sources_file_.c_str(), base_dir_.c_str(), + kUserHostDst, NULL}; EXPECT_TRUE(Parse(static_cast(std::size(argv)) - 1, argv, ¶meters_)); EXPECT_TRUE(parameters_.options.relative); ExpectNoError(); } TEST_F(ParamsTest, FilesFrom_WithoutSourceArg) { - const char* argv[] = { - "cdc_rsync.exe", "--ip=1.2.3.4", "--port=1234", "--files-from", - sources_file_.c_str(), "destination", NULL}; + const char* argv[] = {"cdc_rsync.exe", "--files-from", sources_file_.c_str(), + kUserHostDst, NULL}; EXPECT_TRUE(Parse(static_cast(std::size(argv)) - 1, argv, ¶meters_)); - EXPECT_TRUE(parameters_.sources_dir.empty()); - EXPECT_EQ(parameters_.destination, "destination"); + EXPECT_TRUE(parameters_.options.sources_dir.empty()); + EXPECT_EQ(parameters_.user_host, kUserHost); + EXPECT_EQ(parameters_.destination, kDst); ExpectNoError(); } TEST_F(ParamsTest, FilesFrom_WithSourceArg) { - const char* argv[] = { - "cdc_rsync.exe", "--ip=1.2.3.4", "--port=1234", "--files-from", - sources_file_.c_str(), base_dir_.c_str(), "destination", NULL}; + const char* argv[] = {"cdc_rsync.exe", "--files-from", + sources_file_.c_str(), base_dir_.c_str(), + kUserHostDst, NULL}; EXPECT_TRUE(Parse(static_cast(std::size(argv)) - 1, argv, ¶meters_)); std::string expected_sources_dir = base_dir_; path::EnsureEndsWithPathSeparator(&expected_sources_dir); - EXPECT_EQ(parameters_.sources_dir, expected_sources_dir); - EXPECT_EQ(parameters_.destination, "destination"); + EXPECT_EQ(parameters_.options.sources_dir, expected_sources_dir); + EXPECT_EQ(parameters_.user_host, kUserHost); + EXPECT_EQ(parameters_.destination, kDst); ExpectNoError(); } TEST_F(ParamsTest, FilesFrom_ParsesFile) { - const char* argv[] = { - "cdc_rsync.exe", "--ip=1.2.3.4", "--port=1234", "--files-from", - sources_file_.c_str(), "destination", NULL}; + const char* argv[] = {"cdc_rsync.exe", "--files-from", sources_file_.c_str(), + kUserHostDst, NULL}; EXPECT_TRUE(Parse(static_cast(std::size(argv)) - 1, argv, ¶meters_)); std::vector expected = {"file1", "file2", "file3"}; @@ -394,13 +440,8 @@ TEST_F(ParamsTest, FilesFrom_ParsesFile) { } TEST_F(ParamsTest, FilesFrom_EmptyFile_WithoutSourceArg) { - const char* argv[] = {"cdc_rsync.exe", - "--ip=1.2.3.4", - "--port=1234", - "--files-from", - empty_sources_file_.c_str(), - "destination", - NULL}; + const char* argv[] = {"cdc_rsync.exe", "--files-from", + empty_sources_file_.c_str(), kUserHostDst, NULL}; EXPECT_FALSE( Parse(static_cast(std::size(argv)) - 1, argv, ¶meters_)); ExpectError(empty_sources_file_); @@ -408,14 +449,9 @@ TEST_F(ParamsTest, FilesFrom_EmptyFile_WithoutSourceArg) { } TEST_F(ParamsTest, FilesFrom_EmptyFile_WithSourceArg) { - const char* argv[] = {"cdc_rsync.exe", - "--ip=1.2.3.4", - "--port=1234", - "--files-from", - empty_sources_file_.c_str(), - base_dir_.c_str(), - "destination", - NULL}; + const char* argv[] = { + "cdc_rsync.exe", "--files-from", empty_sources_file_.c_str(), + base_dir_.c_str(), kUserHostDst, NULL}; EXPECT_FALSE( Parse(static_cast(std::size(argv)) - 1, argv, ¶meters_)); ExpectError(empty_sources_file_); @@ -423,17 +459,16 @@ TEST_F(ParamsTest, FilesFrom_EmptyFile_WithSourceArg) { } TEST_F(ParamsTest, FilesFrom_NoDestination) { - const char* argv[] = {"cdc_rsync.exe", "--ip=1.2.3.4", "--port=1234", - "--files-from", sources_file_.c_str(), NULL}; + const char* argv[] = {"cdc_rsync.exe", "--files-from", sources_file_.c_str(), + NULL}; EXPECT_FALSE( Parse(static_cast(std::size(argv)) - 1, argv, ¶meters_)); ExpectError("Missing destination"); } TEST_F(ParamsTest, IncludeFrom_NoFile) { - const char* argv[] = { - "cdc_rsync.exe", "--ip=1.2.3.4", "--port=1234", "source", - "destination", "--include-from", NULL}; + const char* argv[] = {"cdc_rsync.exe", kSrc, kUserHostDst, "--include-from", + NULL}; EXPECT_FALSE( Parse(static_cast(std::size(argv)) - 1, argv, ¶meters_)); ExpectError(NeedsValueError("include-from")); @@ -441,20 +476,22 @@ TEST_F(ParamsTest, IncludeFrom_NoFile) { TEST_F(ParamsTest, IncludeFrom_ParsesFile) { std::string file = path::Join(base_dir_, "include_files.txt"); - const char* argv[] = { - "cdc_rsync.exe", "--ip=1.2.3.4", "--port=1234", "--include-from", - file.c_str(), "source", "destination", NULL}; + const char* argv[] = {"cdc_rsync.exe", "--include-from", + file.c_str(), kSrc, + kUserHostDst, NULL}; EXPECT_TRUE(Parse(static_cast(std::size(argv)) - 1, argv, ¶meters_)); - ASSERT_EQ(parameters_.filter_rules.size(), 1); - ASSERT_EQ(parameters_.filter_rules[0].type, FilterRule::Type::kInclude); - ASSERT_EQ(parameters_.filter_rules[0].pattern, "file3"); + const std::vector& rules = + parameters_.options.filter.GetRules(); + ASSERT_EQ(rules.size(), 1); + ASSERT_EQ(rules[0].type, PathFilter::Rule::Type::kInclude); + ASSERT_EQ(rules[0].pattern, "file3"); ExpectNoError(); } TEST_F(ParamsTest, ExcludeFrom_NoFile) { - const char* argv[] = {"cdc_rsync.exe", "source", "destination", - "--exclude-from", NULL}; + const char* argv[] = {"cdc_rsync.exe", kSrc, kUserHostDst, "--exclude-from", + NULL}; EXPECT_FALSE( Parse(static_cast(std::size(argv)) - 1, argv, ¶meters_)); ExpectError(NeedsValueError("exclude-from")); @@ -462,16 +499,18 @@ TEST_F(ParamsTest, ExcludeFrom_NoFile) { TEST_F(ParamsTest, ExcludeFrom_ParsesFile) { std::string file = path::Join(base_dir_, "exclude_files.txt"); - const char* argv[] = { - "cdc_rsync.exe", "--ip=1.2.3.4", "--port=1234", "--exclude-from", - file.c_str(), "source", "destination", NULL}; + const char* argv[] = {"cdc_rsync.exe", "--exclude-from", + file.c_str(), kSrc, + kUserHostDst, NULL}; EXPECT_TRUE(Parse(static_cast(std::size(argv)) - 1, argv, ¶meters_)); - ASSERT_EQ(parameters_.filter_rules.size(), 2); - EXPECT_EQ(parameters_.filter_rules[0].type, FilterRule::Type::kExclude); - EXPECT_EQ(parameters_.filter_rules[0].pattern, "file1"); - EXPECT_EQ(parameters_.filter_rules[1].type, FilterRule::Type::kExclude); - EXPECT_EQ(parameters_.filter_rules[1].pattern, "file2"); + const std::vector& rules = + parameters_.options.filter.GetRules(); + ASSERT_EQ(rules.size(), 2); + EXPECT_EQ(rules[0].type, PathFilter::Rule::Type::kExclude); + EXPECT_EQ(rules[0].pattern, "file1"); + EXPECT_EQ(rules[1].type, PathFilter::Rule::Type::kExclude); + EXPECT_EQ(rules[1].pattern, "file2"); ExpectNoError(); } @@ -479,31 +518,31 @@ TEST_F(ParamsTest, IncludeExcludeMixed_ProperOrder) { std::string exclude_file = path::Join(base_dir_, "exclude_files.txt"); std::string include_file = path::Join(base_dir_, "include_files.txt"); const char* argv[] = {"cdc_rsync.exe", - "--ip=1.2.3.4", - "--port=1234", "--include-from", include_file.c_str(), "--exclude=excl1", - "source", + kSrc, "--exclude-from", exclude_file.c_str(), - "destination", + kUserHostDst, "--include", "incl1", NULL}; EXPECT_TRUE(Parse(static_cast(std::size(argv)) - 1, argv, ¶meters_)); - ASSERT_EQ(parameters_.filter_rules.size(), 5); - EXPECT_EQ(parameters_.filter_rules[0].type, FilterRule::Type::kInclude); - EXPECT_EQ(parameters_.filter_rules[0].pattern, "file3"); - EXPECT_EQ(parameters_.filter_rules[1].type, FilterRule::Type::kExclude); - EXPECT_EQ(parameters_.filter_rules[1].pattern, "excl1"); - EXPECT_EQ(parameters_.filter_rules[2].type, FilterRule::Type::kExclude); - EXPECT_EQ(parameters_.filter_rules[2].pattern, "file1"); - EXPECT_EQ(parameters_.filter_rules[3].type, FilterRule::Type::kExclude); - EXPECT_EQ(parameters_.filter_rules[3].pattern, "file2"); - EXPECT_EQ(parameters_.filter_rules[4].type, FilterRule::Type::kInclude); - EXPECT_EQ(parameters_.filter_rules[4].pattern, "incl1"); + const std::vector& rules = + parameters_.options.filter.GetRules(); + ASSERT_EQ(rules.size(), 5); + EXPECT_EQ(rules[0].type, PathFilter::Rule::Type::kInclude); + EXPECT_EQ(rules[0].pattern, "file3"); + EXPECT_EQ(rules[1].type, PathFilter::Rule::Type::kExclude); + EXPECT_EQ(rules[1].pattern, "excl1"); + EXPECT_EQ(rules[2].type, PathFilter::Rule::Type::kExclude); + EXPECT_EQ(rules[2].pattern, "file1"); + EXPECT_EQ(rules[3].type, PathFilter::Rule::Type::kExclude); + EXPECT_EQ(rules[3].pattern, "file2"); + EXPECT_EQ(rules[4].type, PathFilter::Rule::Type::kInclude); + EXPECT_EQ(rules[4].pattern, "incl1"); ExpectNoError(); } diff --git a/cdc_rsync_cli/testdata/params/empty_source_files.txt b/cdc_rsync/testdata/params/empty_source_files.txt similarity index 100% rename from cdc_rsync_cli/testdata/params/empty_source_files.txt rename to cdc_rsync/testdata/params/empty_source_files.txt diff --git a/cdc_rsync_cli/testdata/params/exclude_files.txt b/cdc_rsync/testdata/params/exclude_files.txt similarity index 100% rename from cdc_rsync_cli/testdata/params/exclude_files.txt rename to cdc_rsync/testdata/params/exclude_files.txt diff --git a/cdc_rsync_cli/testdata/params/include_files.txt b/cdc_rsync/testdata/params/include_files.txt similarity index 100% rename from cdc_rsync_cli/testdata/params/include_files.txt rename to cdc_rsync/testdata/params/include_files.txt diff --git a/cdc_rsync_cli/testdata/params/source_files.txt b/cdc_rsync/testdata/params/source_files.txt similarity index 100% rename from cdc_rsync_cli/testdata/params/source_files.txt rename to cdc_rsync/testdata/params/source_files.txt diff --git a/cdc_rsync_cli/.gitignore b/cdc_rsync_cli/.gitignore deleted file mode 100644 index 7dc8dde..0000000 --- a/cdc_rsync_cli/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -x64/* -*.log -*.user \ No newline at end of file diff --git a/cdc_rsync_cli/BUILD b/cdc_rsync_cli/BUILD deleted file mode 100644 index 79ec2af..0000000 --- a/cdc_rsync_cli/BUILD +++ /dev/null @@ -1,44 +0,0 @@ -package(default_visibility = [ - "//:__subpackages__", -]) - -cc_binary( - name = "cdc_rsync", - srcs = ["main.cc"], - deps = [ - ":params", - "//cdc_rsync", - ], -) - -cc_library( - name = "params", - srcs = ["params.cc"], - hdrs = ["params.h"], - deps = [ - "//cdc_rsync", - "@com_github_zstd//:zstd", - "@com_google_absl//absl/status", - ], -) - -cc_test( - name = "params_test", - srcs = ["params_test.cc"], - data = ["testdata/root.txt"] + glob(["testdata/params/**"]), - deps = [ - ":params", - "//common:test_main", - "@com_google_googletest//:gtest", - ], -) - -filegroup( - name = "all_test_sources", - srcs = glob(["*_test.cc"]), -) - -filegroup( - name = "all_test_data", - srcs = glob(["testdata/**"]), -) diff --git a/cdc_rsync_cli/main.cc b/cdc_rsync_cli/main.cc deleted file mode 100644 index a308428..0000000 --- a/cdc_rsync_cli/main.cc +++ /dev/null @@ -1,72 +0,0 @@ -// Copyright 2022 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#define WIN32_LEAN_AND_MEAN -#include - -#include -#include - -#include "cdc_rsync/cdc_rsync.h" -#include "cdc_rsync_cli/params.h" -#include "common/util.h" - -int wmain(int argc, wchar_t* argv[]) { - cdc_ft::params::Parameters parameters; - - // Convert args from wide to UTF8 strings. - std::vector utf8_str_args; - utf8_str_args.reserve(argc); - for (int i = 0; i < argc; i++) { - utf8_str_args.push_back(cdc_ft::Util::WideToUtf8Str(argv[i])); - } - - // Convert args from UTF8 strings to UTF8 c-strings. - std::vector utf8_args; - utf8_args.reserve(argc); - for (const auto& utf8_str_arg : utf8_str_args) { - utf8_args.push_back(utf8_str_arg.c_str()); - } - - if (!cdc_ft::params::Parse(argc, utf8_args.data(), ¶meters)) { - return 1; - } - - // Convert sources from string-vec to c-str-vec. - std::vector sources_ptr; - sources_ptr.reserve(parameters.sources.size()); - for (const std::string& source : parameters.sources) { - sources_ptr.push_back(source.c_str()); - } - - // Convert filter rules from string-structs to c-str-structs. - std::vector filter_rules; - filter_rules.reserve(parameters.filter_rules.size()); - for (const cdc_ft::params::Parameters::FilterRule& rule : - parameters.filter_rules) { - filter_rules.emplace_back(rule.type, rule.pattern.c_str()); - } - - const char* error_message = nullptr; - cdc_ft::ReturnCode code = cdc_ft::Sync( - ¶meters.options, filter_rules.data(), parameters.filter_rules.size(), - parameters.sources_dir.c_str(), sources_ptr.data(), - parameters.sources.size(), parameters.destination.c_str(), - &error_message); - - if (error_message) { - fprintf(stderr, "Error: %s\n", error_message); - } - return static_cast(code); -} diff --git a/cdc_rsync_cli/testdata/root.txt b/cdc_rsync_cli/testdata/root.txt deleted file mode 100644 index e69de29..0000000 diff --git a/cdc_rsync_server/cdc_rsync_server.cc b/cdc_rsync_server/cdc_rsync_server.cc index 07058a6..c167c5e 100644 --- a/cdc_rsync_server/cdc_rsync_server.cc +++ b/cdc_rsync_server/cdc_rsync_server.cc @@ -146,14 +146,14 @@ PathFilter::Rule::Type ToInternalType( } // namespace -GgpRsyncServer::GgpRsyncServer() = default; +CdcRsyncServer::CdcRsyncServer() = default; -GgpRsyncServer::~GgpRsyncServer() { +CdcRsyncServer::~CdcRsyncServer() { message_pump_.reset(); socket_.reset(); } -bool GgpRsyncServer::CheckComponents( +bool CdcRsyncServer::CheckComponents( const std::vector& components) { // Components are expected to reside in the same dir as the executable. std::string component_dir; @@ -172,7 +172,7 @@ bool GgpRsyncServer::CheckComponents( return true; } -absl::Status GgpRsyncServer::Run(int port) { +absl::Status CdcRsyncServer::Run(int port) { socket_ = std::make_unique(); absl::Status status = socket_->StartListening(port); if (!status.ok()) { @@ -205,7 +205,7 @@ absl::Status GgpRsyncServer::Run(int port) { return absl::OkStatus(); } -absl::Status GgpRsyncServer::Sync() { +absl::Status CdcRsyncServer::Sync() { // First, the client sends us options, e.g. the |destination_| directory. absl::Status status = HandleSetOptions(); if (!status.ok()) { @@ -281,7 +281,7 @@ absl::Status GgpRsyncServer::Sync() { return absl::OkStatus(); } -absl::Status GgpRsyncServer::HandleSetOptions() { +absl::Status CdcRsyncServer::HandleSetOptions() { LOG_INFO("Receiving options"); SetOptionsRequest request; @@ -324,7 +324,7 @@ absl::Status GgpRsyncServer::HandleSetOptions() { return absl::OkStatus(); } -absl::Status GgpRsyncServer::FindFiles() { +absl::Status CdcRsyncServer::FindFiles() { Stopwatch stopwatch; FileFinder finder; @@ -350,7 +350,7 @@ absl::Status GgpRsyncServer::FindFiles() { return absl::OkStatus(); } -absl::Status GgpRsyncServer::HandleSendAllFiles() { +absl::Status CdcRsyncServer::HandleSendAllFiles() { std::string current_directory; for (;;) { @@ -385,7 +385,7 @@ absl::Status GgpRsyncServer::HandleSendAllFiles() { } } -absl::Status GgpRsyncServer::DiffFiles() { +absl::Status CdcRsyncServer::DiffFiles() { LOG_INFO("Diffing files"); // Be sure to move the data. It can grow quite large with millions of files. @@ -412,7 +412,7 @@ absl::Status GgpRsyncServer::DiffFiles() { return absl::OkStatus(); } -absl::Status GgpRsyncServer::RemoveExtraneousFilesAndDirs() { +absl::Status CdcRsyncServer::RemoveExtraneousFilesAndDirs() { FileDeleterAndSender deleter(message_pump_.get()); // To guarantee that the folders are empty before they are removed, files are @@ -451,7 +451,7 @@ absl::Status GgpRsyncServer::RemoveExtraneousFilesAndDirs() { return absl::OkStatus(); } -absl::Status GgpRsyncServer::CreateMissingDirs() { +absl::Status CdcRsyncServer::CreateMissingDirs() { for (const DirInfo& dir : diff_.missing_dirs) { // Make directory. std::string path = path::Join(destination_, dir.filepath); @@ -475,7 +475,7 @@ absl::Status GgpRsyncServer::CreateMissingDirs() { } template -absl::Status GgpRsyncServer::SendFileIndices(const char* file_type, +absl::Status CdcRsyncServer::SendFileIndices(const char* file_type, const std::vector& files) { LOG_INFO("Sending indices of missing files to client"); constexpr char error_fmt[] = "Failed to send indices of %s files."; @@ -516,7 +516,7 @@ absl::Status GgpRsyncServer::SendFileIndices(const char* file_type, return absl::OkStatus(); } -absl::Status GgpRsyncServer::HandleSendMissingFileData() { +absl::Status CdcRsyncServer::HandleSendMissingFileData() { if (diff_.missing_files.empty()) { return absl::OkStatus(); } @@ -641,7 +641,7 @@ absl::Status GgpRsyncServer::HandleSendMissingFileData() { return absl::OkStatus(); } -absl::Status GgpRsyncServer::SyncChangedFiles() { +absl::Status CdcRsyncServer::SyncChangedFiles() { if (diff_.changed_files.empty()) { return absl::OkStatus(); } @@ -729,7 +729,7 @@ absl::Status GgpRsyncServer::SyncChangedFiles() { return absl::OkStatus(); } -absl::Status GgpRsyncServer::HandleShutdown() { +absl::Status CdcRsyncServer::HandleShutdown() { ShutdownRequest request; absl::Status status = message_pump_->ReceiveMessage(PacketType::kShutdown, &request); @@ -746,7 +746,7 @@ absl::Status GgpRsyncServer::HandleShutdown() { return absl::OkStatus(); } -void GgpRsyncServer::Thread_OnPackageReceived(PacketType type) { +void CdcRsyncServer::Thread_OnPackageReceived(PacketType type) { if (type != PacketType::kToggleCompression) { return; } diff --git a/cdc_rsync_server/cdc_rsync_server.h b/cdc_rsync_server/cdc_rsync_server.h index 7a75684..0a58549 100644 --- a/cdc_rsync_server/cdc_rsync_server.h +++ b/cdc_rsync_server/cdc_rsync_server.h @@ -33,10 +33,10 @@ namespace cdc_ft { class MessagePump; class ServerSocket; -class GgpRsyncServer { +class CdcRsyncServer { public: - GgpRsyncServer(); - ~GgpRsyncServer(); + CdcRsyncServer(); + ~CdcRsyncServer(); // Checks that the gamelet components (cdc_rsync_server binary etc.) are // up-to-date by checking their sizes and timestamps. diff --git a/cdc_rsync_server/main.cc b/cdc_rsync_server/main.cc index 667ab93..0fef17d 100644 --- a/cdc_rsync_server/main.cc +++ b/cdc_rsync_server/main.cc @@ -48,7 +48,6 @@ ServerExitCode GetExitCode(const absl::Status& status) { case Tag::kSocketEof: // Usually means client disconnected and shut down already. case Tag::kDeployServer: - case Tag::kInstancePickerNotAvailableInQuietMode: case Tag::kConnectionTimeout: case Tag::kCount: // Should not happen in server. @@ -82,7 +81,7 @@ int main(int argc, const char** argv) { cdc_ft::Log::Initialize( std::make_unique(cdc_ft::LogLevel::kWarning)); - cdc_ft::GgpRsyncServer server; + cdc_ft::CdcRsyncServer server; if (!server.CheckComponents(components)) { return cdc_ft::kServerExitCodeOutOfDate; } diff --git a/common/port_manager.h b/common/port_manager.h index 5267e75..34f0a4b 100644 --- a/common/port_manager.h +++ b/common/port_manager.h @@ -51,10 +51,14 @@ class PortManager { // Reserves a port in the range passed to the constructor. The port is // released automatically upon destruction if ReleasePort() is not called // explicitly. - // |timeout_sec| is the timeout for finding available ports on the gamelet - // instance. Returns a DeadlineExceeded error if the timeout is exceeded. + // |check_remote| determines whether the remote port should be checked as + // well. If false, the check is skipped and a port might be returned that is + // still in use remotely. + // |remote_timeout_sec| is the timeout for finding available ports on the + // remote instance. Not used if |check_remote| is false. + // Returns a DeadlineExceeded error if the timeout is exceeded. // Returns a ResourceExhausted error if no ports are available. - absl::StatusOr ReservePort(int timeout_sec); + absl::StatusOr ReservePort(bool check_remote, int remote_timeout_sec); // Releases a reserved port. absl::Status ReleasePort(int port); diff --git a/common/port_manager_test.cc b/common/port_manager_test.cc index ba2d56d..2ec1cd6 100644 --- a/common/port_manager_test.cc +++ b/common/port_manager_test.cc @@ -25,8 +25,8 @@ namespace cdc_ft { namespace { -constexpr int kGameletPort = 12345; -constexpr char kGameletIp[] = "1.2.3.4"; +constexpr int kSshPort = 12345; +constexpr char kUserHost[] = "user@1.2.3.4"; constexpr char kGuid[] = "f77bcdfe-368c-4c45-9f01-230c5e7e2132"; constexpr int kFirstPort = 44450; @@ -38,6 +38,9 @@ constexpr int kTimeoutSec = 1; constexpr char kLocalNetstat[] = "netstat -a -n -p tcp"; constexpr char kRemoteNetstat[] = "netstat --numeric --listening --tcp"; +constexpr bool kCheckRemote = true; +constexpr bool kNoCheckRemote = false; + constexpr char kLocalNetstatOutFmt[] = "TCP 127.0.0.1:50000 127.0.0.1:%i ESTABLISHED"; constexpr char kRemoteNetstatOutFmt[] = @@ -53,7 +56,7 @@ class PortManagerTest : public ::testing::Test { void SetUp() override { Log::Initialize(std::make_unique(LogLevel::kInfo)); - remote_util_.SetIpAndPort(kGameletIp, kGameletPort); + remote_util_.SetUserHostAndPort(kUserHost, kSshPort); } void TearDown() override { Log::Shutdown(); } @@ -70,7 +73,16 @@ TEST_F(PortManagerTest, ReservePortSuccess) { process_factory_.SetProcessOutput(kLocalNetstat, "", "", 0); process_factory_.SetProcessOutput(kRemoteNetstat, "", "", 0); - absl::StatusOr port = port_manager_.ReservePort(kTimeoutSec); + absl::StatusOr port = + port_manager_.ReservePort(kCheckRemote, kTimeoutSec); + ASSERT_OK(port); + EXPECT_EQ(*port, kFirstPort); +} + +TEST_F(PortManagerTest, ReservePortNoRemoteSuccess) { + process_factory_.SetProcessOutput(kLocalNetstat, "", "", 0); + + absl::StatusOr port = port_manager_.ReservePort(kNoCheckRemote, 0); ASSERT_OK(port); EXPECT_EQ(*port, kFirstPort); } @@ -83,7 +95,8 @@ TEST_F(PortManagerTest, ReservePortAllLocalPortsTaken) { process_factory_.SetProcessOutput(kLocalNetstat, local_netstat_out, "", 0); process_factory_.SetProcessOutput(kRemoteNetstat, "", "", 0); - absl::StatusOr port = port_manager_.ReservePort(kTimeoutSec); + absl::StatusOr port = + port_manager_.ReservePort(kCheckRemote, kTimeoutSec); EXPECT_TRUE(absl::IsResourceExhausted(port.status())); EXPECT_TRUE( absl::StrContains(port.status().message(), "No port available in range")); @@ -97,7 +110,8 @@ TEST_F(PortManagerTest, ReservePortAllRemotePortsTaken) { process_factory_.SetProcessOutput(kLocalNetstat, "", "", 0); process_factory_.SetProcessOutput(kRemoteNetstat, remote_netstat_out, "", 0); - absl::StatusOr port = port_manager_.ReservePort(kTimeoutSec); + absl::StatusOr port = + port_manager_.ReservePort(kCheckRemote, kTimeoutSec); EXPECT_TRUE(absl::IsResourceExhausted(port.status())); EXPECT_TRUE( absl::StrContains(port.status().message(), "No port available in range")); @@ -107,7 +121,8 @@ TEST_F(PortManagerTest, ReservePortLocalNetstatFails) { process_factory_.SetProcessOutput(kLocalNetstat, "", "", 1); process_factory_.SetProcessOutput(kRemoteNetstat, "", "", 0); - absl::StatusOr port = port_manager_.ReservePort(kTimeoutSec); + absl::StatusOr port = + port_manager_.ReservePort(kCheckRemote, kTimeoutSec); EXPECT_NOT_OK(port); EXPECT_TRUE( absl::StrContains(port.status().message(), @@ -118,7 +133,8 @@ TEST_F(PortManagerTest, ReservePortRemoteNetstatFails) { process_factory_.SetProcessOutput(kLocalNetstat, "", "", 0); process_factory_.SetProcessOutput(kRemoteNetstat, "", "", 1); - absl::StatusOr port = port_manager_.ReservePort(kTimeoutSec); + absl::StatusOr port = + port_manager_.ReservePort(kCheckRemote, kTimeoutSec); EXPECT_NOT_OK(port); EXPECT_TRUE(absl::StrContains(port.status().message(), "Failed to find available ports on instance")); @@ -129,7 +145,8 @@ TEST_F(PortManagerTest, ReservePortRemoteNetstatTimesOut) { process_factory_.SetProcessNeverExits(kRemoteNetstat); steady_clock_.AutoAdvance(kTimeoutSec * 2 * 1000); - absl::StatusOr port = port_manager_.ReservePort(kTimeoutSec); + absl::StatusOr port = + port_manager_.ReservePort(kCheckRemote, kTimeoutSec); EXPECT_NOT_OK(port); EXPECT_TRUE(absl::IsDeadlineExceeded(port.status())); EXPECT_TRUE(absl::StrContains(port.status().message(), @@ -146,10 +163,14 @@ TEST_F(PortManagerTest, ReservePortMultipleInstances) { // Port managers use shared memory, so different instances know about each // other. This would even work if |port_manager_| and |port_manager2| belonged // to different processes, but we don't test that here. - EXPECT_EQ(*port_manager_.ReservePort(kTimeoutSec), kFirstPort + 0); - EXPECT_EQ(*port_manager2.ReservePort(kTimeoutSec), kFirstPort + 1); - EXPECT_EQ(*port_manager_.ReservePort(kTimeoutSec), kFirstPort + 2); - EXPECT_EQ(*port_manager2.ReservePort(kTimeoutSec), kFirstPort + 3); + EXPECT_EQ(*port_manager_.ReservePort(kCheckRemote, kTimeoutSec), + kFirstPort + 0); + EXPECT_EQ(*port_manager2.ReservePort(kCheckRemote, kTimeoutSec), + kFirstPort + 1); + EXPECT_EQ(*port_manager_.ReservePort(kCheckRemote, kTimeoutSec), + kFirstPort + 2); + EXPECT_EQ(*port_manager2.ReservePort(kCheckRemote, kTimeoutSec), + kFirstPort + 3); } TEST_F(PortManagerTest, ReservePortReusesPortsInLRUOrder) { @@ -157,7 +178,7 @@ TEST_F(PortManagerTest, ReservePortReusesPortsInLRUOrder) { process_factory_.SetProcessOutput(kRemoteNetstat, "", "", 0); for (int n = 0; n < kNumPorts * 2; ++n) { - EXPECT_EQ(*port_manager_.ReservePort(kTimeoutSec), + EXPECT_EQ(*port_manager_.ReservePort(kCheckRemote, kTimeoutSec), kFirstPort + n % kNumPorts); system_clock_.Advance(1000); } @@ -167,10 +188,11 @@ TEST_F(PortManagerTest, ReleasePort) { process_factory_.SetProcessOutput(kLocalNetstat, "", "", 0); process_factory_.SetProcessOutput(kRemoteNetstat, "", "", 0); - absl::StatusOr port = port_manager_.ReservePort(kTimeoutSec); + absl::StatusOr port = + port_manager_.ReservePort(kCheckRemote, kTimeoutSec); EXPECT_EQ(*port, kFirstPort); EXPECT_OK(port_manager_.ReleasePort(*port)); - port = port_manager_.ReservePort(kTimeoutSec); + port = port_manager_.ReservePort(kCheckRemote, kTimeoutSec); EXPECT_EQ(*port, kFirstPort); } @@ -180,10 +202,13 @@ TEST_F(PortManagerTest, ReleasePortOnDestruction) { auto port_manager2 = std::make_unique( kGuid, kFirstPort, kLastPort, &process_factory_, &remote_util_); - EXPECT_EQ(*port_manager2->ReservePort(kTimeoutSec), kFirstPort + 0); - EXPECT_EQ(*port_manager_.ReservePort(kTimeoutSec), kFirstPort + 1); + EXPECT_EQ(*port_manager2->ReservePort(kCheckRemote, kTimeoutSec), + kFirstPort + 0); + EXPECT_EQ(*port_manager_.ReservePort(kCheckRemote, kTimeoutSec), + kFirstPort + 1); port_manager2.reset(); - EXPECT_EQ(*port_manager_.ReservePort(kTimeoutSec), kFirstPort + 0); + EXPECT_EQ(*port_manager_.ReservePort(kCheckRemote, kTimeoutSec), + kFirstPort + 0); } TEST_F(PortManagerTest, FindAvailableLocalPortsSuccess) { diff --git a/common/port_manager_win.cc b/common/port_manager_win.cc index 30093fd..9192456 100644 --- a/common/port_manager_win.cc +++ b/common/port_manager_win.cc @@ -121,7 +121,8 @@ PortManager::~PortManager() { } } -absl::StatusOr PortManager::ReservePort(int timeout_sec) { +absl::StatusOr PortManager::ReservePort(bool check_remote, + int remote_timeout_sec) { // Find available port on workstation. std::unordered_set local_ports; ASSIGN_OR_RETURN(local_ports, @@ -129,13 +130,16 @@ absl::StatusOr PortManager::ReservePort(int timeout_sec) { process_factory_, false), "Failed to find available ports on workstation"); - // Find available port on remote gamelet. - std::unordered_set remote_ports; - ASSIGN_OR_RETURN(remote_ports, - FindAvailableRemotePorts(first_port_, last_port_, "0.0.0.0", - process_factory_, remote_util_, - timeout_sec, false, steady_clock_), - "Failed to find available ports on instance"); + // Find available port on remote instance. + std::unordered_set remote_ports = local_ports; + if (check_remote) { + ASSIGN_OR_RETURN( + remote_ports, + FindAvailableRemotePorts(first_port_, last_port_, "0.0.0.0", + process_factory_, remote_util_, + remote_timeout_sec, false, steady_clock_), + "Failed to find available ports on instance"); + } // Fetch shared memory. void* mem; diff --git a/common/process_win.cc b/common/process_win.cc index b78f8fd..42eb62b 100644 --- a/common/process_win.cc +++ b/common/process_win.cc @@ -58,8 +58,8 @@ absl::Status CreatePipeForOverlappedIo(ScopedHandle* pipe_read_end, ScopedHandle* pipe_write_end) { // We need named pipes for overlapped IO, so create a unique name. int id = g_pipe_serial_number++; - std::string pipe_name = absl::StrFormat( - R"(\\.\Pipe\GgpRsyncIoPipe.%08x.%08x)", GetCurrentProcessId(), id); + std::string pipe_name = absl::StrFormat(R"(\\.\Pipe\CdcIoPipe.%08x.%08x)", + GetCurrentProcessId(), id); // Set the bInheritHandle flag so pipe handles are inherited. SECURITY_ATTRIBUTES security_attributes; diff --git a/common/remote_util.cc b/common/remote_util.cc index 1e85eac..cf5b557 100644 --- a/common/remote_util.cc +++ b/common/remote_util.cc @@ -14,10 +14,9 @@ #include "common/remote_util.h" -#include #include -#include +#include "absl/strings/str_cat.h" #include "absl/strings/str_format.h" #include "common/path.h" #include "common/status.h" @@ -25,32 +24,6 @@ namespace cdc_ft { namespace { -// Escapes command line argument for the Microsoft command line parser in -// preparation for quoting. Double quotes are backslash-escaped. Literal -// backslashes are backslash-escaped if they are followed by a double quote, or -// if they are part of a sequence of backslashes that are followed by a double -// quote. -std::string EscapeForWindows(const std::string& argument) { - std::string str = - std::regex_replace(argument, std::regex(R"(\\*(?=""|$))"), "$1$1"); - return std::regex_replace(str, std::regex("\""), "\\\""); -} - -// Quotes and escapes a command line argument following the convention -// understood by the Microsoft command line parser. -std::string QuoteArgument(const std::string& argument) { - return absl::StrFormat("\"%s\"", EscapeForWindows(argument)); -} - -// Quotes and escapes a command line arguments for use in ssh command. The -// argument is first escaped and quoted for Linux using single quotes and then -// it is escaped to be used by the Microsoft command line parser. -std::string QuoteAndEscapeArgumentForSsh(const std::string& argument) { - std::string quoted_argument = absl::StrFormat( - "'%s'", std::regex_replace(argument, std::regex("'"), "'\\''")); - return EscapeForWindows(quoted_argument); -} - // Gets the argument for SSH (reverse) port forwarding, e.g. -L23:localhost:45. std::string GetPortForwardingArg(int local_port, int remote_port, bool reverse) { @@ -69,21 +42,29 @@ RemoteUtil::RemoteUtil(int verbosity, bool quiet, process_factory_(process_factory), forward_output_to_log_(forward_output_to_log) {} -void RemoteUtil::SetIpAndPort(const std::string& gamelet_ip, int ssh_port) { - gamelet_ip_ = gamelet_ip; - ssh_port_ = ssh_port; +void RemoteUtil::SetUserHostAndPort(std::string user_host, int port) { + user_host_ = std::move(user_host); + ssh_port_ = port; +} +void RemoteUtil::SetScpCommand(std::string scp_command) { + scp_command_ = std::move(scp_command); +} + +void RemoteUtil::SetSshCommand(std::string ssh_command) { + ssh_command_ = std::move(ssh_command); } absl::Status RemoteUtil::Scp(std::vector source_filepaths, const std::string& dest, bool compress) { - absl::Status status = CheckIpPort(); + absl::Status status = CheckHostPort(); if (!status.ok()) { return status; } std::string source_args; for (const std::string& sourceFilePath : source_filepaths) { - source_args += QuoteArgument(sourceFilePath) + " "; + // Workaround for scp thinking that C is a host in C:\path\to\foo. + source_args += QuoteArgument("//./" + sourceFilePath) + " "; } // -p preserves timestamps. This enables timestamp-based up-to-date checks. @@ -91,18 +72,10 @@ absl::Status RemoteUtil::Scp(std::vector source_filepaths, start_info.command = absl::StrFormat( "%s " "%s %s -p -T " - "-F %s " - "-i %s -P %i " - "-oStrictHostKeyChecking=yes " - "-oUserKnownHostsFile=\"\"\"%s\"\"\" %s " - "cloudcast@%s:" + "-P %i %s " "%s", - QuoteArgument(sdk_util_.GetScpExePath()), - quiet_ || verbosity_ < 2 ? "-q" : "", compress ? "-C" : "", - QuoteArgument(sdk_util_.GetSshConfigPath()), - QuoteArgument(sdk_util_.GetSshKeyFilePath()), ssh_port_, - sdk_util_.GetSshKnownHostsFilePath(), source_args, - QuoteArgument(gamelet_ip_), QuoteAndEscapeArgumentForSsh(dest)); + scp_command_, quiet_ || verbosity_ < 2 ? "-q" : "", compress ? "-C" : "", + ssh_port_, source_args, QuoteArgument(user_host_ + ":" + dest)); start_info.name = "scp"; start_info.forward_output_to_log = forward_output_to_log_; @@ -111,7 +84,7 @@ absl::Status RemoteUtil::Scp(std::vector source_filepaths, absl::Status RemoteUtil::Sync(std::vector source_filepaths, const std::string& dest) { - absl::Status status = CheckIpPort(); + absl::Status status = CheckHostPort(); if (!status.ok()) { return status; } @@ -123,9 +96,9 @@ absl::Status RemoteUtil::Sync(std::vector source_filepaths, ProcessStartInfo start_info; start_info.command = absl::StrFormat( - "%s --ip=%s --port=%i -z %s %s%s", - path::Join(sdk_util_.GetDevBinPath(), "cdc_rsync"), - QuoteArgument(gamelet_ip_), ssh_port_, + "cdc_rsync --ip=%s --port=%i -z " + "%s %s%s", + QuoteArgument(user_host_), ssh_port_, quiet_ || verbosity_ < 2 ? "-q " : " ", source_args, QuoteArgument(dest)); start_info.name = "cdc_rsync"; start_info.forward_output_to_log = forward_output_to_log_; @@ -135,16 +108,16 @@ absl::Status RemoteUtil::Sync(std::vector source_filepaths, absl::Status RemoteUtil::Chmod(const std::string& mode, const std::string& remote_path, bool quiet) { - std::string remote_command = absl::StrFormat( - "chmod %s %s %s", QuoteArgument(mode), - QuoteAndEscapeArgumentForSsh(remote_path), quiet ? "-f" : ""); + std::string remote_command = + absl::StrFormat("chmod %s %s %s", QuoteArgument(mode), + EscapeForWindows(remote_path), quiet ? "-f" : ""); return Run(remote_command, "chmod"); } absl::Status RemoteUtil::Rm(const std::string& remote_path, bool force) { - std::string remote_command = absl::StrFormat( - "rm %s %s", force ? "-f" : "", QuoteAndEscapeArgumentForSsh(remote_path)); + std::string remote_command = absl::StrFormat("rm %s %s", force ? "-f" : "", + EscapeForWindows(remote_path)); return Run(remote_command, "rm"); } @@ -152,14 +125,14 @@ absl::Status RemoteUtil::Rm(const std::string& remote_path, bool force) { absl::Status RemoteUtil::Mv(const std::string& old_remote_path, const std::string& new_remote_path) { std::string remote_command = - absl::StrFormat("mv %s %s", QuoteAndEscapeArgumentForSsh(old_remote_path), - QuoteAndEscapeArgumentForSsh(new_remote_path)); + absl::StrFormat("mv %s %s", EscapeForWindows(old_remote_path), + EscapeForWindows(new_remote_path)); return Run(remote_command, "mv"); } absl::Status RemoteUtil::Run(std::string remote_command, std::string name) { - absl::Status status = CheckIpPort(); + absl::Status status = CheckHostPort(); if (!status.ok()) { return status; } @@ -201,25 +174,37 @@ ProcessStartInfo RemoteUtil::BuildProcessStartInfoForSshInternal( start_info.command = absl::StrFormat( "%s " "%s -tt " - "-F %s " - "-i %s " "-oServerAliveCountMax=6 " // Number of lost msgs before ssh terminates "-oServerAliveInterval=5 " // Time interval between alive msgs - "-oStrictHostKeyChecking=yes " - "-oUserKnownHostsFile=\"\"\"%s\"\"\" %s" - "cloudcast@%s -p %i %s", - QuoteArgument(sdk_util_.GetSshExePath()), - quiet_ || verbosity_ < 2 ? "-q" : "", - QuoteArgument(sdk_util_.GetSshConfigPath()), - QuoteArgument(sdk_util_.GetSshKeyFilePath()), - sdk_util_.GetSshKnownHostsFilePath(), forward_arg, - QuoteArgument(gamelet_ip_), ssh_port_, remote_command_arg); + "%s %s -p %i %s", + ssh_command_, quiet_ || verbosity_ < 2 ? "-q" : "", forward_arg, + QuoteArgument(user_host_), ssh_port_, remote_command_arg); start_info.forward_output_to_log = forward_output_to_log_; return start_info; } -absl::Status RemoteUtil::CheckIpPort() { - if (gamelet_ip_.empty() || ssh_port_ == 0) { +std::string RemoteUtil::EscapeForWindows(const std::string& argument) { + std::string str = + std::regex_replace(argument, std::regex(R"(\\*(?="|$))"), "$&$&"); + return std::regex_replace(str, std::regex(R"(")"), R"(\")"); +} + +std::string RemoteUtil::QuoteArgument(const std::string& argument) { + return absl::StrCat("\"", EscapeForWindows(argument), "\""); +} + +std::string RemoteUtil::QuoteArgumentForSsh(const std::string& argument) { + return absl::StrFormat( + "'%s'", std::regex_replace(argument, std::regex("'"), "'\\''")); +} + +std::string RemoteUtil::QuoteAndEscapeArgumentForSsh( + const std::string& argument) { + return EscapeForWindows(QuoteArgumentForSsh(argument)); +} + +absl::Status RemoteUtil::CheckHostPort() { + if (user_host_.empty() || ssh_port_ == 0) { return MakeStatus("IP or port not set"); } diff --git a/common/remote_util.h b/common/remote_util.h index 936ebdb..31e59b3 100644 --- a/common/remote_util.h +++ b/common/remote_util.h @@ -22,7 +22,6 @@ #include "absl/status/status.h" #include "common/process.h" -#include "common/sdk_util.h" namespace cdc_ft { @@ -30,6 +29,8 @@ namespace cdc_ft { // Windows-only. class RemoteUtil { public: + static constexpr int kDefaultSshPort = 22; + // If |verbosity| is > 0 and |quiet| is false, output from scp, ssh etc. // commands is shown. // If |quiet| is true, scp, ssh etc. commands use quiet mode. @@ -38,61 +39,67 @@ class RemoteUtil { RemoteUtil(int verbosity, bool quiet, ProcessFactory* process_factory, bool forward_output_to_log); - // Returns the initialization status. Should be OK unless in case of some rare - // internal error. Should be checked before accessing any members. - const absl::Status& GetInitStatus() const { - return sdk_util_.GetInitStatus(); - } + // Sets the SSH username and hostname of the remote instance, as well as the + // SSH tunnel port. |user_host| must be of the form [user@]host. + void SetUserHostAndPort(std::string user_host, int port); - // Set IP of the remote instance and the ssh tunnel port. - void SetIpAndPort(const std::string& gamelet_ip, int ssh_port); + // Sets the SCP command binary path and additional arguments, e.g. + // C:\path\to\scp.exe -F -i + // -oStrictHostKeyChecking=yes -oUserKnownHostsFile="""file""" + // By default, searches scp.exe on the path environment variables. + void SetScpCommand(std::string scp_command); + + // Sets the SSH command binary path and additional arguments, e.g. + // C:\path\to\ssh.exe -F -i + // -oStrictHostKeyChecking=yes -oUserKnownHostsFile="""file""" + // By default, searches ssh.exe on the path environment variables. + void SetSshCommand(std::string ssh_command); // Copies |source_filepaths| to the remote folder |dest| on the gamelet using - // scp. Must call either InitSsh or SetGameletIp before calling this method. - // If |compress| is true, compressed upload is used. + // scp. If |compress| is true, compressed upload is used. + // Must call SetUserHostAndPort before calling this method. absl::Status Scp(std::vector source_filepaths, const std::string& dest, bool compress); // Syncs |source_filepaths| to the remote folder |dest| on the gamelet using - // cdc_rsync. Must call either InitSsh or SetGameletIp before calling this - // method. + // cdc_rsync. Must call SetUserHostAndPort before calling this method. absl::Status Sync(std::vector source_filepaths, const std::string& dest); // Calls 'chmod |mode| |remote_path|' on the gamelet. - // Must call either InitSsh or SetGameletIp before calling this method. + // Must call SetUserHostAndPort before calling this method. absl::Status Chmod(const std::string& mode, const std::string& remote_path, bool quiet = false); // Calls 'rm [-f] |remote_path|' on the gamelet. - // Must call either InitSsh or SetGameletIp before calling this method. + // Must call SetUserHostAndPort before calling this method. absl::Status Rm(const std::string& remote_path, bool force); // Calls `mv |old_remote_path| |new_remote_path| on the gamelet. - // Must call either InitSsh or SetGameletIp before calling this method. + // Must call SetUserHostAndPort before calling this method. absl::Status Mv(const std::string& old_remote_path, const std::string& new_remote_path); // Runs |remote_command| on the gamelet. The command must be properly escaped. // |name| is the name of the command displayed in the logs. - // Must call either InitSsh or SetGameletIp before calling this method. + // Must call SetUserHostAndPort before calling this method. absl::Status Run(std::string remote_command, std::string name); - // Builds an ssh command that executes |remote_command| on the gamelet. + // Builds an SSH command that executes |remote_command| on the gamelet. ProcessStartInfo BuildProcessStartInfoForSsh(std::string remote_command); - // Builds an ssh command that runs SSH port forwarding to the gamelet, using + // Builds an SSH command that runs SSH port forwarding to the gamelet, using // the given |local_port| and |remote_port|. // If |reverse| is true, sets up reverse port forwarding. - // Must call either InitSsh or SetGameletIp before calling this method. + // Must call SetUserHostAndPort before calling this method. ProcessStartInfo BuildProcessStartInfoForSshPortForward(int local_port, int remote_port, bool reverse); - // Builds an ssh command that executes |remote_command| on the gamelet, using + // Builds an SSH command that executes |remote_command| on the gamelet, using // port forwarding with given |local_port| and |remote_port|. // If |reverse| is true, sets up reverse port forwarding. - // Must call either InitSsh or SetGameletIp before calling this method. + // Must call SetUserHostAndPort before calling this method. ProcessStartInfo BuildProcessStartInfoForSshPortForwardAndCommand( int local_port, int remote_port, bool reverse, std::string remote_command); @@ -100,9 +107,28 @@ class RemoteUtil { // Returns whether output is suppressed. bool Quiet() const { return quiet_; } + // Escapes command line argument for the Microsoft command line parser in + // preparation for quoting. Double quotes are backslash-escaped. One or more + // backslashes are backslash-escaped if they are followed by a double quote, + // or if they occur at the end of the string, e.g. + // foo\bar -> foo\bar, foo\ -> foo\\, foo\\"bar -> foo\\\\\"bar. + static std::string EscapeForWindows(const std::string& argument); + + // Quotes and escapes a command line argument following the convention + // understood by the Microsoft command line parser. + static std::string QuoteArgument(const std::string& argument); + + // Quotes and escapes a command line argument for usage in SSH. + static std::string QuoteArgumentForSsh(const std::string& argument); + + // Quotes and escapes a command line arguments for use in SSH command. The + // argument is first escaped and quoted for Linux using single quotes and then + // it is escaped to be used by the Microsoft command line parser. + static std::string QuoteAndEscapeArgumentForSsh(const std::string& argument); + private: - // Verifies that both |gamelet_ip_| and |ssh_port_| are set. - absl::Status CheckIpPort(); + // Verifies that both || and |ssh_port_| are set. + absl::Status CheckHostPort(); // Common code for BuildProcessStartInfoForSsh*. ProcessStartInfo BuildProcessStartInfoForSshInternal( @@ -113,9 +139,10 @@ class RemoteUtil { ProcessFactory* const process_factory_; const bool forward_output_to_log_; - SdkUtil sdk_util_; - std::string gamelet_ip_; - int ssh_port_ = 0; + std::string scp_command_ = "scp"; + std::string ssh_command_ = "ssh"; + std::string user_host_; + int ssh_port_ = kDefaultSshPort; }; } // namespace cdc_ft diff --git a/common/remote_util_test.cc b/common/remote_util_test.cc index 3befbce..7ac730a 100644 --- a/common/remote_util_test.cc +++ b/common/remote_util_test.cc @@ -21,11 +21,11 @@ namespace cdc_ft { namespace { -constexpr int kGameletPort = 12345; -constexpr char kGameletPortArg[] = "-p 12345"; +constexpr int kSshPort = 12345; +constexpr char kSshPortArg[] = "-p 12345"; -constexpr char kGameletIp[] = "1.2.3.4"; -constexpr char kGameletIpArg[] = "cloudcast@\"1.2.3.4\""; +constexpr char kUserHost[] = "user@example.com"; +constexpr char kUserHostArg[] = "\"user@example.com\""; constexpr int kLocalPort = 23456; constexpr int kRemotePort = 34567; @@ -44,7 +44,7 @@ class RemoteUtilTest : public ::testing::Test { void SetUp() override { Log::Initialize(std::make_unique(LogLevel::kInfo)); - util_.SetIpAndPort(kGameletIp, kGameletPort); + util_.SetUserHostAndPort(kUserHost, kSshPort); } void TearDown() override { Log::Shutdown(); } @@ -64,42 +64,68 @@ class RemoteUtilTest : public ::testing::Test { TEST_F(RemoteUtilTest, BuildProcessStartInfoForSsh) { ProcessStartInfo si = util_.BuildProcessStartInfoForSsh(kCommand); - ExpectContains(si.command, - {"ssh.exe", "GGP\\ssh\\id", "oStrictHostKeyChecking=yes", - "oUserKnownHostsFile", "known_hosts", kGameletPortArg, - kGameletIpArg, kCommand}); + ExpectContains(si.command, {"ssh", kSshPortArg, kUserHostArg, kCommand}); } TEST_F(RemoteUtilTest, BuildProcessStartInfoForSshPortForward) { ProcessStartInfo si = util_.BuildProcessStartInfoForSshPortForward( kLocalPort, kRemotePort, kRegular); ExpectContains(si.command, - {"ssh.exe", "GGP\\ssh\\id", "oStrictHostKeyChecking=yes", - "oUserKnownHostsFile", "known_hosts", kGameletPortArg, - kGameletIpArg, kPortForwardingArg}); + {"ssh", kSshPortArg, kUserHostArg, kPortForwardingArg}); si = util_.BuildProcessStartInfoForSshPortForward(kLocalPort, kRemotePort, kReverse); ExpectContains(si.command, - {"ssh.exe", "GGP\\ssh\\id", "oStrictHostKeyChecking=yes", - "oUserKnownHostsFile", "known_hosts", kGameletPortArg, - kGameletIpArg, kReversePortForwardingArg}); + {"ssh", kSshPortArg, kUserHostArg, kReversePortForwardingArg}); } TEST_F(RemoteUtilTest, BuildProcessStartInfoForSshPortForwardAndCommand) { ProcessStartInfo si = util_.BuildProcessStartInfoForSshPortForwardAndCommand( kLocalPort, kRemotePort, kRegular, kCommand); - ExpectContains(si.command, - {"ssh.exe", "GGP\\ssh\\id", "oStrictHostKeyChecking=yes", - "oUserKnownHostsFile", "known_hosts", kGameletPortArg, - kGameletIpArg, kPortForwardingArg, kCommand}); + ExpectContains(si.command, {"ssh", kSshPortArg, kUserHostArg, + kPortForwardingArg, kCommand}); si = util_.BuildProcessStartInfoForSshPortForwardAndCommand( kLocalPort, kRemotePort, kReverse, kCommand); - ExpectContains(si.command, - {"ssh.exe", "GGP\\ssh\\id", "oStrictHostKeyChecking=yes", - "oUserKnownHostsFile", "known_hosts", kGameletPortArg, - kGameletIpArg, kReversePortForwardingArg, kCommand}); + ExpectContains(si.command, {"ssh", kSshPortArg, kUserHostArg, + kReversePortForwardingArg, kCommand}); +} +TEST_F(RemoteUtilTest, BuildProcessStartInfoForSshWithCustomCommand) { + constexpr char kCustomSshCmd[] = "C:\\path\\to\\ssh.exe --fooarg --bararg=42"; + util_.SetSshCommand(kCustomSshCmd); + ProcessStartInfo si = util_.BuildProcessStartInfoForSsh(kCommand); + ExpectContains(si.command, {kCustomSshCmd}); +} + +TEST_F(RemoteUtilTest, EscapeForWindows) { + EXPECT_EQ("foo", RemoteUtil::EscapeForWindows("foo")); + EXPECT_EQ("foo bar", RemoteUtil::EscapeForWindows("foo bar")); + EXPECT_EQ("foo\\bar", RemoteUtil::EscapeForWindows("foo\\bar")); + EXPECT_EQ("\\\\foo", RemoteUtil::EscapeForWindows("\\\\foo")); + EXPECT_EQ("foo\\\\", RemoteUtil::EscapeForWindows("foo\\")); + EXPECT_EQ("foo\\\\\\\\", RemoteUtil::EscapeForWindows("foo\\\\")); + EXPECT_EQ("foo\\\"", RemoteUtil::EscapeForWindows("foo\"")); + EXPECT_EQ("foo\\\"bar", RemoteUtil::EscapeForWindows("foo\"bar")); + EXPECT_EQ("foo\\\\\\\"bar", RemoteUtil::EscapeForWindows("foo\\\"bar")); + EXPECT_EQ("foo\\\\\\\\\\\"bar", RemoteUtil::EscapeForWindows("foo\\\\\"bar")); + EXPECT_EQ("\\\"foo\\\"", RemoteUtil::EscapeForWindows("\"foo\"")); + EXPECT_EQ("\\\" \\file.txt", RemoteUtil::EscapeForWindows("\" \\file.txt")); +} + +TEST_F(RemoteUtilTest, QuoteArgument) { + EXPECT_EQ("\"foo\"", RemoteUtil::QuoteArgument("foo")); + EXPECT_EQ("\"foo bar\"", RemoteUtil::QuoteArgument("foo bar")); + EXPECT_EQ("\"foo\\bar\"", RemoteUtil::QuoteArgument("foo\\bar")); + EXPECT_EQ("\"\\\\foo\"", RemoteUtil::QuoteArgument("\\\\foo")); + EXPECT_EQ("\"foo\\\\\"", RemoteUtil::QuoteArgument("foo\\")); + EXPECT_EQ("\"foo\\\\\\\\\"", RemoteUtil::QuoteArgument("foo\\\\")); + EXPECT_EQ("\"foo\\\"\"", RemoteUtil::QuoteArgument("foo\"")); + EXPECT_EQ("\"foo\\\"bar\"", RemoteUtil::QuoteArgument("foo\"bar")); + EXPECT_EQ("\"foo\\\\\\\"bar\"", RemoteUtil::QuoteArgument("foo\\\"bar")); + EXPECT_EQ("\"foo\\\\\\\\\\\"bar\"", + RemoteUtil::QuoteArgument("foo\\\\\"bar")); + EXPECT_EQ("\"\\\"foo\\\"\"", RemoteUtil::QuoteArgument("\"foo\"")); + EXPECT_EQ("\"\\\" \\file.txt\"", RemoteUtil::QuoteArgument("\" \\file.txt")); } } // namespace diff --git a/common/sdk_util.cc b/common/sdk_util.cc index fc9cb1c..453c497 100644 --- a/common/sdk_util.cc +++ b/common/sdk_util.cc @@ -31,13 +31,6 @@ SdkUtil::SdkUtil() { absl::Status status = path::GetEnv("GGP_SDK_PATH", &ggp_sdk_path_env_); if (absl::IsNotFound(status) || ggp_sdk_path_env_.empty()) ggp_sdk_path_env_ = path::Join(program_files_path_, "GGP SDK"); - - // Create an empty config file if it does not exist yet. - const std::string ssh_config_path = GetSshConfigPath(); - if (!path::Exists(ssh_config_path)) { - init_status_.Update(path::CreateDirRec(path::DirName(ssh_config_path))); - init_status_.Update(path::WriteFile(ssh_config_path, nullptr, 0)); - } } SdkUtil::~SdkUtil() = default; @@ -57,37 +50,8 @@ std::string SdkUtil::GetLogPath(const char* log_base_name) const { return path::Join(GetUserConfigPath(), "logs", log_base_name + timestamp_ext); } -std::string SdkUtil::GetSshConfigPath() const { - return path::Join(GetUserConfigPath(), "ssh", "config"); -} - -std::string SdkUtil::GetSshKeyFilePath() const { - return path::Join(GetUserConfigPath(), "ssh", "id_rsa"); -} - -std::string SdkUtil::GetSshKnownHostsFilePath() const { - return path::Join(GetUserConfigPath(), "ssh", "known_hosts"); -} - -std::string SdkUtil::GetSDKPath() const { - assert(init_status_.ok()); - return ggp_sdk_path_env_; -} - std::string SdkUtil::GetDevBinPath() const { - return path::Join(GetSDKPath(), "dev", "bin"); -} - -std::string SdkUtil::GetSshPath() const { - return path::Join(GetSDKPath(), "tools", "OpenSSH-Win64"); -} - -std::string SdkUtil::GetSshExePath() const { - return path::Join(GetSshPath(), "ssh.exe"); -} - -std::string SdkUtil::GetScpExePath() const { - return path::Join(GetSshPath(), "scp.exe"); + return path::Join(ggp_sdk_path_env_, "dev", "bin"); } } // namespace cdc_ft diff --git a/common/sdk_util.h b/common/sdk_util.h index b42c036..5b2cbe3 100644 --- a/common/sdk_util.h +++ b/common/sdk_util.h @@ -51,38 +51,10 @@ class SdkUtil { // %APPDATA%\GGP\logs\log_base_name.20210729-125930.log. std::string GetLogPath(const char* log_base_name) const; - // Returns the path of the ssh configuration file, e.g. - // %APPDATA%\GGP\ssh\config. - std::string GetSshConfigPath() const; - - // Returns the path of the ssh private key file in the SDK configuration, e.g. - // %APPDATA%\GGP\ssh\id_rsa. - std::string GetSshKeyFilePath() const; - - // Returns the path of the ssh known_hosts file in the SDK configuration, e.g. - // %APPDATA%\GGP\ssh\known_hosts. - std::string GetSshKnownHostsFilePath() const; - - // Returns the path of the installed SDK, e.g. - // C:\Program Files\GGP SDK. - std::string GetSDKPath() const; - // Returns the path of the dev tools that ship with the SDK, e.g. // C:\Program Files\GGP SDK\dev\bin. std::string GetDevBinPath() const; - // Returns the path of the OpenSSH tools that ship with the SDK, e.g. - // C:\Program Files\GGP SDK\tools\OpenSSH-Win64. - std::string GetSshPath() const; - - // Returns the path of ssh.exe that ships with the SDK, e.g. - // C:\Program Files\GGP SDK\tools\OpenSSH-Win64\ssh.exe. - std::string GetSshExePath() const; - - // Returns the path of scp.exe that ships with the SDK, e.g. - // C:\Program Files\GGP SDK\tools\OpenSSH-Win64\scp.exe. - std::string GetScpExePath() const; - private: std::string roaming_appdata_path_; std::string program_files_path_; diff --git a/common/sdk_util_test.cc b/common/sdk_util_test.cc index 911834c..53574ee 100644 --- a/common/sdk_util_test.cc +++ b/common/sdk_util_test.cc @@ -42,13 +42,6 @@ class SdkUtilTest : public ::testing::Test { protected: void CheckSdkPaths(const SdkUtil& sdk_util, const std::string& sdk_dir) { - EXPECT_EQ(sdk_util.GetSDKPath(), sdk_dir); - EXPECT_EQ(sdk_util.GetSshPath(), - path::Join(sdk_dir, "tools\\OpenSSH-Win64")); - EXPECT_EQ(sdk_util.GetSshExePath(), - path::Join(sdk_dir, "tools\\OpenSSH-Win64\\ssh.exe")); - EXPECT_EQ(sdk_util.GetScpExePath(), - path::Join(sdk_dir, "tools\\OpenSSH-Win64\\scp.exe")); EXPECT_EQ(sdk_util.GetDevBinPath(), path::Join(sdk_dir, "dev", "bin")); } @@ -81,11 +74,6 @@ TEST_F(SdkUtilTest, CheckRoamingAppDataPaths) { const std::string ggp_path = path::Join(appdata_dir, "GGP"); EXPECT_EQ(sdk_util.GetUserConfigPath(), ggp_path); EXPECT_EQ(sdk_util.GetServicesConfigPath(), path::Join(ggp_path, "services")); - EXPECT_EQ(sdk_util.GetSshConfigPath(), path::Join(ggp_path, "ssh", "config")); - EXPECT_EQ(sdk_util.GetSshKeyFilePath(), - path::Join(ggp_path, "ssh", "id_rsa")); - EXPECT_EQ(sdk_util.GetSshKnownHostsFilePath(), - path::Join(ggp_path, "ssh", "known_hosts")); } TEST_F(SdkUtilTest, CheckSdkPathsWithoutGgpSdkPathEnv) { diff --git a/common/status.h b/common/status.h index 6dce7ff..d2febfa 100644 --- a/common/status.h +++ b/common/status.h @@ -77,14 +77,11 @@ enum class Tag : uint8_t { // The gamelet components need to be re-deployed. kDeployServer = 2, - // Something asks for user input, but we're in quiet mode. - kInstancePickerNotAvailableInQuietMode = 3, - // Timeout while trying to connect to the gamelet component. - kConnectionTimeout = 4, + kConnectionTimeout = 3, // MUST BE LAST. - kCount = 5, + kCount = 4, }; // Tags a status. No-op if |status| is OK. Overwrites existing tags. diff --git a/file_transfer.sln b/file_transfer.sln index 573245c..755fce1 100644 --- a/file_transfer.sln +++ b/file_transfer.sln @@ -4,7 +4,7 @@ VisualStudioVersion = 16.0.31702.278 MinimumVisualStudioVersion = 10.0.40219.1 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "CDC RSync", "CDC RSync", "{74FA49B8-56C3-4F9E-B9D5-35FA1C9A13C8}" EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "cdc_rsync_cli", "cdc_rsync_cli\cdc_rsync_cli.vcxproj", "{3FAC852A-00A8-4CFB-9160-07EFF2B73562}" +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "cdc_rsync", "cdc_rsync\cdc_rsync.vcxproj", "{3FAC852A-00A8-4CFB-9160-07EFF2B73562}" EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "cdc_rsync_server", "cdc_rsync_server\cdc_rsync_server.vcxproj", "{4ECE65E0-D950-4B96-8AD5-0313261B8C8D}" EndProject diff --git a/tests_cdc_rsync/BUILD b/tests_cdc_rsync/BUILD index 833ca53..010c668 100644 --- a/tests_cdc_rsync/BUILD +++ b/tests_cdc_rsync/BUILD @@ -15,24 +15,22 @@ cc_binary( srcs = [ "//cdc_rsync:all_test_sources", "//cdc_rsync/base:all_test_sources", - "//cdc_rsync_cli:all_test_sources", "//cdc_rsync_server:all_test_sources", ], data = [ "//cdc_rsync:all_test_data", "//cdc_rsync/base:all_test_data", - "//cdc_rsync_cli:all_test_data", "//cdc_rsync_server:all_test_data", ], deps = [ "//cdc_rsync:file_finder_and_sender", "//cdc_rsync:parallel_file_opener", + "//cdc_rsync:params", "//cdc_rsync:progress_tracker", "//cdc_rsync:zstd_stream", "//cdc_rsync/base:cdc_interface", "//cdc_rsync/base:fake_socket", "//cdc_rsync/base:message_pump", - "//cdc_rsync_cli:params", "//cdc_rsync_server:file_deleter_and_sender", "//cdc_rsync_server:file_diff_generator", "//cdc_rsync_server:file_finder", diff --git a/tools/windows_cc_library.bzl b/tools/windows_cc_library.bzl deleted file mode 100644 index 0129b00..0000000 --- a/tools/windows_cc_library.bzl +++ /dev/null @@ -1,91 +0,0 @@ -# Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -""" -This is a simple cc_windows_shared_library rule for builing a DLL on Windows -that other cc rules can depend on. - -Example useage: - cc_windows_shared_library( - name = "hellolib", - srcs = [ - "hello-library.cpp", - ], - hdrs = ["hello-library.h"], - # Use this to distinguish compiling vs. linking against the DLL. - copts = ["/DCOMPILING_DLL"], - ) - -Define COMPILING_DLL to export symbols in the header when compiling the DLL as -follows: - - #ifdef COMPILING_DLL - #define DLLEXPORT __declspec(dllexport) - #else - #define DLLEXPORT __declspec(dllimport) - #endif - - DLLEXPORT void foo(); - -For more information and sample usage, see: -https://github.com/bazelbuild/bazel/blob/master/examples/windows/dll/ -""" - -load("@rules_cc//cc:defs.bzl", "cc_binary", "cc_import", "cc_library") - -def cc_windows_shared_library( - name, - srcs = [], - deps = [], - hdrs = [], - visibility = None, - **kwargs): - """A simple cc_windows_shared_library rule for builing a Windows DLL.""" - dll_name = name + ".dll" - import_lib_name = name + "_import_lib" - import_target_name = name + "_dll_import" - - # Building a shared library requires a cc_binary with linkshared = 1 set. - cc_binary( - name = dll_name, - srcs = srcs + hdrs, - deps = deps, - linkshared = 1, - **kwargs - ) - - # Get the import library for the dll - native.filegroup( - name = import_lib_name, - srcs = [":" + dll_name], - output_group = "interface_library", - ) - - # Because we cannot directly depend on cc_binary from other cc rules in deps attribute, - # we use cc_import as a bridge to depend on the dll. - cc_import( - name = import_target_name, - interface_library = ":" + import_lib_name, - shared_library = ":" + dll_name, - ) - - # Create a new cc_library to also include the headers needed for the shared library - cc_library( - name = name, - hdrs = hdrs, - visibility = visibility, - deps = deps + [ - ":" + import_target_name, - ], - )