From 9fdccb3548246fb9f2afa5668c4c4070ccd38a43 Mon Sep 17 00:00:00 2001 From: ljusten Date: Tue, 15 Nov 2022 12:48:09 +0100 Subject: [PATCH] Remove GGP dependencies from CDC RSync (#1) * Remove dependencies of cdc_sync from GGP Allows overriding the SSH and SCP commands via command line flags. Hence, strict host checking, SSH config etc. can be removed since it is passed in by command line flags for GGP. Also deploys cdc_rsync_server to ~/.cache/cdc_file_transfer/ and creates that dir if it does not exist. * Tweak RemoteUtil Replaces localhost: by //./ in the workaround for scp since localhost: had two disadvantages: 1) It required 2 gnubby touches for gLinux and 2) it didn't work for ggp. //./ works for both. Also tweaks quoting, which didn't quite work for ggp. * Don't check remote ports in cdc_rsync Turns off checking remote ports in PortManager. In the future, the server should return available ports after failing to connect to the provided port. Since now the first remote connection is running cdc_rsync_server, the timeout check has to be done when running that process. * Remove now-unused kInstancePickerNotAvailableInQuietMode enum * Add more details to the readme * [cdc_rsync] Accept [user@]host:destination Removes the --ip command line argument and assumes user/host are passed in along with the destination, so it works in the same way as other popular tools. * [ggp_rsync] Combine server deploy commands Combines two chmod and one mv command into one ssh command. This makes deploy a bit quicker, especially if each ssh command involves touching your gnubby. * Remove GGP specific stuff from VS build commands * [cdc_rsync] Get rid of cdc_rsync.dll Compile the CDC RSync client as a static library instead. This removes quite a bit of boiler plate and makes string handling easier since we can now pass std::strings instead of const chars. Also fixes an issue where we were sometimes trying to assign nullptr to std::strings, which is forbidden. * Allow specifying ssh/scp commands with env vars * Rename GgpRsync* to CdcRsync* * Merge ggp_rsync_cli into ggp_rsync * [cdc_rsync] Refactor cdc_rsync.cc/h Merges cdc_rsync.cc/h with main.cc and CdcRsyncClient since code is closer to where it's being used and should be more readable. --- NMakeBazelProject.targets | 3 +- README.md | 114 +++++- all_files.vcxitems | 14 +- asset_stream_manager/cdc_fuse_manager.cc | 2 +- asset_stream_manager/session.cc | 2 +- cdc_rsync/BUILD | 53 ++- cdc_rsync/cdc_rsync.cc | 125 ------- cdc_rsync/cdc_rsync.h | 107 ------ .../cdc_rsync.vcxproj | 6 +- .../cdc_rsync.vcxproj.filters | 0 cdc_rsync/cdc_rsync_client.cc | 127 ++++--- cdc_rsync/cdc_rsync_client.h | 40 +- cdc_rsync/dllmain.cc | 29 -- cdc_rsync/error_messages.h | 54 --- cdc_rsync/main.cc | 147 ++++++++ {cdc_rsync_cli => cdc_rsync}/params.cc | 159 +++++--- {cdc_rsync_cli => cdc_rsync}/params.h | 24 +- {cdc_rsync_cli => cdc_rsync}/params_test.cc | 353 ++++++++++-------- .../testdata/params/empty_source_files.txt | 0 .../testdata/params/exclude_files.txt | 0 .../testdata/params/include_files.txt | 0 .../testdata/params/source_files.txt | 0 cdc_rsync_cli/.gitignore | 3 - cdc_rsync_cli/BUILD | 44 --- cdc_rsync_cli/main.cc | 72 ---- cdc_rsync_cli/testdata/root.txt | 0 cdc_rsync_server/cdc_rsync_server.cc | 32 +- cdc_rsync_server/cdc_rsync_server.h | 6 +- cdc_rsync_server/main.cc | 3 +- common/port_manager.h | 10 +- common/port_manager_test.cc | 63 +++- common/port_manager_win.cc | 20 +- common/process_win.cc | 4 +- common/remote_util.cc | 123 +++--- common/remote_util.h | 79 ++-- common/remote_util_test.cc | 72 ++-- common/sdk_util.cc | 38 +- common/sdk_util.h | 28 -- common/sdk_util_test.cc | 12 - common/status.h | 7 +- file_transfer.sln | 2 +- tests_cdc_rsync/BUILD | 4 +- tools/windows_cc_library.bzl | 91 ----- 43 files changed, 952 insertions(+), 1120 deletions(-) delete mode 100644 cdc_rsync/cdc_rsync.cc delete mode 100644 cdc_rsync/cdc_rsync.h rename cdc_rsync_cli/cdc_rsync_cli.vcxproj => cdc_rsync/cdc_rsync.vcxproj (98%) rename cdc_rsync_cli/cdc_rsync_cli.vcxproj.filters => cdc_rsync/cdc_rsync.vcxproj.filters (100%) delete mode 100644 cdc_rsync/dllmain.cc delete mode 100644 cdc_rsync/error_messages.h create mode 100644 cdc_rsync/main.cc rename {cdc_rsync_cli => cdc_rsync}/params.cc (74%) rename {cdc_rsync_cli => cdc_rsync}/params.h (64%) rename {cdc_rsync_cli => cdc_rsync}/params_test.cc (55%) rename {cdc_rsync_cli => cdc_rsync}/testdata/params/empty_source_files.txt (100%) rename {cdc_rsync_cli => cdc_rsync}/testdata/params/exclude_files.txt (100%) rename {cdc_rsync_cli => cdc_rsync}/testdata/params/include_files.txt (100%) rename {cdc_rsync_cli => cdc_rsync}/testdata/params/source_files.txt (100%) delete mode 100644 cdc_rsync_cli/.gitignore delete mode 100644 cdc_rsync_cli/BUILD delete mode 100644 cdc_rsync_cli/main.cc delete mode 100644 cdc_rsync_cli/testdata/root.txt delete mode 100644 tools/windows_cc_library.bzl diff --git a/NMakeBazelProject.targets b/NMakeBazelProject.targets index e194096..8c10aad 100644 --- a/NMakeBazelProject.targets +++ b/NMakeBazelProject.targets @@ -32,8 +32,7 @@ | sed -r "s/^([^:\(]+[:\(][[:digit:]]+(,[[:digit:]]+)?[:\)])/$(BazelSourcePathPrefix)\\1/" 2>&1 $(BazelSedCommand) - - --config=$(BazelPlatform) --workspace_status_command="exit 0" --bes_backend= + --config=$(BazelPlatform) $(BazelArgs) --linkopt=-Wl,--strip-all $(BazelArgs) --distinct_host_configuration=false diff --git a/README.md b/README.md index a0a3239..292901a 100644 --- a/README.md +++ b/README.md @@ -6,9 +6,113 @@ on Content Defined Chunking (CDC), in particular to split up files into chunks. ## CDC RSync -Tool to sync files to a remote machine, similar to the standard Linux -[rsync](https://linux.die.net/man/1/rsync). It supports fast compression and -uses a higher performing remote diffing approach based on CDC. -## Asset Streaming -Tool to stream assets from a Windows machine to a Linux device. \ No newline at end of file +CDC RSync is a tool to sync files from a Windows machine to a Linux device, +similar to the standard Linux [rsync](https://linux.die.net/man/1/rsync). It is +basically a copy tool, but optimized for the case where there is already an old +version of the files available in the target directory. +* It skips files quickly if timestamp and file size match. +* It uses fast compression for all data transfer. +* If a file changed, it determines which parts changed and only transfers the + differences. + +The remote diffing algorithm is based on CDC. In our tests, it is up to 30x +faster than the one used in rsync (1500 MB/s vs 50 MB/s). + +## CDC Stream + +CDC Stream is a tool to stream files and directories from a Windows machine to a +Linux device. Conceptually, it is similar to [sshfs](https://github.com/libfuse/sshfs), +but it is optimized for read speed. +* It caches streamed data on the Linux device. +* If a file is re-read on Linux after it changed on Windows, only the + differences are streamed again. The rest is read from cache. +* Stat operations are very fast since the directory metadata (filenames, + permissions etc.) is provided in a streaming-friendly way. + +To efficiently determine which parts of a file changed, the tool uses the same +CDC-based diffing algorithm as CDC RSync. Changes to Windows files are almost +immediately reflected on Linux, with a delay of roughly (0.5s + 0.7s x total +size of changed files in GB). + +The tool does not support writing files back from Linux to Windows; the Linux +directory is readonly. + +# Getting Started + +The project has to be built both on Windows and Linux. + +## Prerequisites + +The following steps have to be executed on **both Windows and Linux**. + +* Download and install Bazel from https://bazel.build/install. +* Clone the repository. + ``` + git clone https://github.com/google/cdc-file-transfer + ``` +* Initialize submodules. + ``` + cd cdc-file-transfer + git submodule update --init --recursive + ``` + +Finally, install an SSH client on the Windows device if not present. +The file transfer tools require `ssh.exe` and `scp.exe`. + +## Building + +The two tools can be built and used independently. + +### CDC Sync + +* Build Linux components + ``` + bazel build --config linux --compilation_mode=opt //cdc_rsync_server + ``` +* Build Windows components + ``` + bazel build --config windows --compilation_mode=opt //cdc_rsync + ``` +* Copy the Linux build output file `cdc_rsync_server` from + `bazel-bin/cdc_rsync_server` on the Linux system to `bazel-bin\cdc_rsync` + on the Windows machine. + +### CDC Stream + +* Build Linux components + ``` + bazel build --config linux --compilation_mode=opt //cdc_fuse_fs + ``` +* Build Windows components + ``` + bazel build --config windows --compilation_mode=opt //asset_stream_manager + ``` +* Copy the Linux build output files `cdc_fuse_fs` and `libfuse.so` from + `bazel-bin/cdc_fuse_fs` on the Linux system to `bazel-bin\asset_stream_manager` + on the Windows machine. + +## Usage + +### CDC Sync +To copy the contents of the Windows directory `C:\path\to\assets` to `~/assets` +on the Linux device `linux.machine.com`, run +``` +cdc_rsync --ssh-command=C:\path\to\ssh.exe --scp-command=C:\path\to\scp.exe C:\path\to\assets\* user@linux.machine.com:~/assets -vr +``` +Depending on your setup, you may have to specify additional arguments for the +ssh and scp commands, including proper quoting, e.g. +``` +cdc_rsync --ssh-command="\"C:\path with space\to\ssh.exe\" -F ssh_config_file -i id_rsa_file -oStrictHostKeyChecking=yes -oUserKnownHostsFile=\"\"\"known_hosts_file\"\"\"" --scp-command="\"C:\path with space\to\scp.exe\" -F ssh_config_file -i id_rsa_file -oStrictHostKeyChecking=yes -oUserKnownHostsFile=\"\"\"known_hosts_file\"\"\"" C:\path\to\assets\* user@linux.machine.com:~/assets -vr +``` +Lengthy ssh/scp commands that rarely change can also be put into environment +variables `CDC_SSH_COMMAND` and `CDC_SCP_COMMAND`, e.g. +``` +set CDC_SSH_COMMAND="C:\path with space\to\ssh.exe" -F ssh_config_file -i id_rsa_file -oStrictHostKeyChecking=yes -oUserKnownHostsFile="""known_hosts_file""" + +set CDC_SCP_COMMAND="C:\path with space\to\scp.exe" -F ssh_config_file -i id_rsa_file -oStrictHostKeyChecking=yes -oUserKnownHostsFile="""known_hosts_file""" + +cdc_rsync C:\path\to\assets\* user@linux.machine.com:~/assets -vr +``` + +### CDC Stream diff --git a/all_files.vcxitems b/all_files.vcxitems index 9428041..4dc5840 100644 --- a/all_files.vcxitems +++ b/all_files.vcxitems @@ -96,10 +96,8 @@ - - @@ -107,9 +105,9 @@ - - - + + + @@ -202,14 +200,12 @@ - - - + @@ -247,7 +243,6 @@ - @@ -259,6 +254,7 @@ + diff --git a/asset_stream_manager/cdc_fuse_manager.cc b/asset_stream_manager/cdc_fuse_manager.cc index a43bb23..e4cd4cc 100644 --- a/asset_stream_manager/cdc_fuse_manager.cc +++ b/asset_stream_manager/cdc_fuse_manager.cc @@ -29,7 +29,7 @@ namespace { constexpr char kFuseFilename[] = "cdc_fuse_fs"; constexpr char kLibFuseFilename[] = "libfuse.so"; constexpr char kFuseStdoutPrefix[] = "cdc_fuse_fs_stdout"; -constexpr char kRemoteToolsBinDir[] = "/opt/developer/tools/bin/"; +constexpr char kRemoteToolsBinDir[] = "~/.cache/cdc_file_transfer/"; // Mount point for FUSE on the gamelet. constexpr char kMountDir[] = "/mnt/workstation"; diff --git a/asset_stream_manager/session.cc b/asset_stream_manager/session.cc index 9666aae..77dcab8 100644 --- a/asset_stream_manager/session.cc +++ b/asset_stream_manager/session.cc @@ -51,7 +51,7 @@ Session::Session(std::string instance_id, std::string instance_ip, /*forward_output_to_logging=*/true), metrics_recorder_(std::move(metrics_recorder)) { assert(metrics_recorder_); - remote_util_.SetIpAndPort(instance_ip, instance_port); + remote_util_.SetUserHostAndPort(instance_ip, instance_port); } Session::~Session() { diff --git a/cdc_rsync/BUILD b/cdc_rsync/BUILD index 1c01242..f0ffa54 100644 --- a/cdc_rsync/BUILD +++ b/cdc_rsync/BUILD @@ -1,12 +1,16 @@ -load( - "//tools:windows_cc_library.bzl", - "cc_windows_shared_library", -) - package(default_visibility = [ "//:__subpackages__", ]) +cc_binary( + name = "cdc_rsync", + srcs = ["main.cc"], + deps = [ + ":cdc_rsync_client", + ":params", + ], +) + cc_library( name = "client_file_info", hdrs = ["client_file_info.h"], @@ -57,25 +61,16 @@ cc_test( ], ) -cc_windows_shared_library( - name = "cdc_rsync", - srcs = [ - "cdc_rsync.cc", - "cdc_rsync_client.cc", - "dllmain.cc", - ], - hdrs = [ - "cdc_rsync.h", - "cdc_rsync_client.h", - "error_messages.h", - ], +cc_library( + name = "cdc_rsync_client", + srcs = ["cdc_rsync_client.cc"], + hdrs = ["cdc_rsync_client.h"], linkopts = select({ "//tools:windows": [ "/DEFAULTLIB:Ws2_32.lib", # Sockets, e.g. recv, send, WSA*. ], "//conditions:default": [], }), - local_defines = ["COMPILING_DLL"], target_compatible_with = ["@platforms//os:windows"], deps = [ ":client_socket", @@ -128,6 +123,28 @@ cc_test( ], ) +cc_library( + name = "params", + srcs = ["params.cc"], + hdrs = ["params.h"], + deps = [ + ":cdc_rsync_client", + "@com_github_zstd//:zstd", + "@com_google_absl//absl/status", + ], +) + +cc_test( + name = "params_test", + srcs = ["params_test.cc"], + data = ["testdata/root.txt"] + glob(["testdata/params/**"]), + deps = [ + ":params", + "//common:test_main", + "@com_google_googletest//:gtest", + ], +) + cc_library( name = "progress_tracker", srcs = ["progress_tracker.cc"], diff --git a/cdc_rsync/cdc_rsync.cc b/cdc_rsync/cdc_rsync.cc deleted file mode 100644 index 1e2d279..0000000 --- a/cdc_rsync/cdc_rsync.cc +++ /dev/null @@ -1,125 +0,0 @@ -// Copyright 2022 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "cdc_rsync/cdc_rsync.h" - -#include - -#include "cdc_rsync/cdc_rsync_client.h" -#include "cdc_rsync/error_messages.h" -#include "common/log.h" -#include "common/path_filter.h" -#include "common/status.h" - -namespace cdc_ft { -namespace { - -ReturnCode TagToMessage(Tag tag, const Options* options, std::string* msg) { - msg->clear(); - switch (tag) { - case Tag::kSocketEof: - *msg = kMsgConnectionLost; - return ReturnCode::kConnectionLost; - - case Tag::kAddressInUse: - *msg = kMsgAddressInUse; - return ReturnCode::kAddressInUse; - - case Tag::kDeployServer: - *msg = kMsgDeployFailed; - return ReturnCode::kDeployFailed; - - case Tag::kInstancePickerNotAvailableInQuietMode: - *msg = kMsgInstancePickerNotAvailableInQuietMode; - return ReturnCode::kInstancePickerNotAvailableInQuietMode; - - case Tag::kConnectionTimeout: - *msg = - absl::StrFormat(kMsgFmtConnectionTimeout, options->ip, options->port); - return ReturnCode::kConnectionTimeout; - - case Tag::kCount: - return ReturnCode::kGenericError; - } - - // Should not happen (TM). Will fall back to status message in this case. - return ReturnCode::kGenericError; -} - -PathFilter::Rule::Type ToInternalType(FilterRule::Type type) { - switch (type) { - case FilterRule::Type::kInclude: - return PathFilter::Rule::Type::kInclude; - case FilterRule::Type::kExclude: - return PathFilter::Rule::Type::kExclude; - } - assert(false); - return PathFilter::Rule::Type::kInclude; -} - -} // namespace - -ReturnCode Sync(const Options* options, const FilterRule* filter_rules, - size_t num_filter_rules, const char* sources_dir, - const char* const* sources, size_t num_sources, - const char* destination, const char** error_message) { - LogLevel log_level = Log::VerbosityToLogLevel(options->verbosity); - Log::Initialize(std::make_unique(log_level)); - - PathFilter path_filter; - for (size_t n = 0; n < num_filter_rules; ++n) { - path_filter.AddRule(ToInternalType(filter_rules[n].type), - filter_rules[n].pattern); - } - - std::vector sources_vec; - for (size_t n = 0; n < num_sources; ++n) { - sources_vec.push_back(sources[n]); - } - - // Run rsync. - GgpRsyncClient client(*options, std::move(path_filter), sources_dir, - std::move(sources_vec), destination); - absl::Status status = client.Run(); - - if (status.ok()) { - *error_message = nullptr; - return ReturnCode::kOk; - } - - std::string msg; - ReturnCode code = ReturnCode::kGenericError; - absl::optional tag = GetTag(status); - if (tag.has_value()) { - code = TagToMessage(tag.value(), options, &msg); - } - - // Fall back to status message. - if (msg.empty()) { - msg = std::string(status.message()); - } else if (options->verbosity >= 2) { - // In verbose mode, log the status as well, so nothing gets lost. - LOG_ERROR("%s", status.ToString().c_str()); - } - - // Store error message in static buffer (don't use std::string through DLL - // boundary!). - static char buf[1024] = {0}; - strncpy_s(buf, msg.c_str(), _TRUNCATE); - *error_message = buf; - - return code; -} - -} // namespace cdc_ft diff --git a/cdc_rsync/cdc_rsync.h b/cdc_rsync/cdc_rsync.h deleted file mode 100644 index 9a13328..0000000 --- a/cdc_rsync/cdc_rsync.h +++ /dev/null @@ -1,107 +0,0 @@ -/* - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef CDC_RSYNC_CDC_RSYNC_H_ -#define CDC_RSYNC_CDC_RSYNC_H_ - -#ifdef COMPILING_DLL -#define CDC_RSYNC_API __declspec(dllexport) -#else -#define CDC_RSYNC_API __declspec(dllimport) -#endif - -namespace cdc_ft { - -#ifdef __cplusplus -extern "C" { -#endif - -struct Options { - const char* ip = nullptr; - int port = 0; - bool delete_ = false; - bool recursive = false; - int verbosity = 0; - bool quiet = false; - bool whole_file = false; - bool relative = false; - bool compress = false; - bool checksum = false; - bool dry_run = false; - bool existing = false; - bool json = false; - const char* copy_dest = nullptr; - int compress_level = 6; - int connection_timeout_sec = 10; - - // Compression level 0 is invalid. - static constexpr int kMinCompressLevel = -5; - static constexpr int kMaxCompressLevel = 22; -}; - -// Rule for including/excluding files. -struct FilterRule { - enum class Type { - kInclude, - kExclude, - }; - - Type type; - const char* pattern; - - FilterRule(Type type, const char* pattern) : type(type), pattern(pattern) {} -}; - -enum class ReturnCode { - // No error. Will match the tool's exit code, so OK must be 0. - kOk = 0, - - // Generic error. - kGenericError = 1, - - // Server connection timed out. - kConnectionTimeout = 2, - - // Connection to the server was shut down unexpectedly. - kConnectionLost = 3, - - // Binding to the forward port failed, probably because there's another - // instance of cdc_rsync running. - kAddressInUse = 4, - - // Server deployment failed. This should be rare, it means that the server - // components were successfully copied, but the up-to-date check still fails. - kDeployFailed = 5, - - // Gamelet selection asks for user input, but we are in quiet mode. - kInstancePickerNotAvailableInQuietMode = 6, -}; - -// Calling Sync() a second time overwrites the data in |error_message|. -CDC_RSYNC_API ReturnCode Sync(const Options* options, - const FilterRule* filter_rules, - size_t filter_num_rules, const char* sources_dir, - const char* const* sources, size_t num_sources, - const char* destination, - const char** error_message); - -#ifdef __cplusplus -} // extern "C" -#endif - -} // namespace cdc_ft - -#endif // CDC_RSYNC_CDC_RSYNC_H_ diff --git a/cdc_rsync_cli/cdc_rsync_cli.vcxproj b/cdc_rsync/cdc_rsync.vcxproj similarity index 98% rename from cdc_rsync_cli/cdc_rsync_cli.vcxproj rename to cdc_rsync/cdc_rsync.vcxproj index b0603e0..c83069a 100644 --- a/cdc_rsync_cli/cdc_rsync_cli.vcxproj +++ b/cdc_rsync/cdc_rsync.vcxproj @@ -42,12 +42,12 @@ - $(SolutionDir)bazel-out\x64_windows-dbg\bin\cdc_rsync_cli\ + $(SolutionDir)bazel-out\x64_windows-dbg\bin\cdc_rsync\ /std:c++17 UNICODE - $(SolutionDir)bazel-out\x64_windows-opt\bin\cdc_rsync_cli\ + $(SolutionDir)bazel-out\x64_windows-opt\bin\cdc_rsync\ UNICODE /std:c++17 @@ -66,7 +66,7 @@ - //cdc_rsync_cli:cdc_rsync + //cdc_rsync cdc_rsync.exe ..\;..\third_party\absl;..\third_party\blake3\c;..\bazel-stadia-file-transfer\external\com_github_zstd\lib;..\third_party\googletest\googletest\include;..\third_party\protobuf\src;$(VC_IncludePath);$(WindowsSDK_IncludePath) ..\/ diff --git a/cdc_rsync_cli/cdc_rsync_cli.vcxproj.filters b/cdc_rsync/cdc_rsync.vcxproj.filters similarity index 100% rename from cdc_rsync_cli/cdc_rsync_cli.vcxproj.filters rename to cdc_rsync/cdc_rsync.vcxproj.filters diff --git a/cdc_rsync/cdc_rsync_client.cc b/cdc_rsync/cdc_rsync_client.cc index c121af1..ea04cab 100644 --- a/cdc_rsync/cdc_rsync_client.cc +++ b/cdc_rsync/cdc_rsync_client.cc @@ -47,7 +47,7 @@ constexpr int kExitCodeNotFound = 127; constexpr int kForwardPortFirst = 44450; constexpr int kForwardPortLast = 44459; constexpr char kGgpServerFilename[] = "cdc_rsync_server"; -constexpr char kRemoteToolsBinDir[] = "/opt/developer/tools/bin/"; +constexpr char kRemoteToolsBinDir[] = "~/.cache/cdc_file_transfer/"; SetOptionsRequest::FilterRule::Type ToProtoType(PathFilter::Rule::Type type) { switch (type) { @@ -94,14 +94,12 @@ absl::Status GetServerExitStatus(int exit_code, const std::string& error_msg) { } // namespace -GgpRsyncClient::GgpRsyncClient(const Options& options, PathFilter path_filter, - std::string sources_dir, +CdcRsyncClient::CdcRsyncClient(const Options& options, std::vector sources, - std::string destination) + std::string user_host, std::string destination) : options_(options), - path_filter_(std::move(path_filter)), - sources_dir_(std::move(sources_dir)), sources_(std::move(sources)), + user_host_(std::move(user_host)), destination_(std::move(destination)), remote_util_(options.verbosity, options.quiet, &process_factory_, /*forward_output_to_log=*/false), @@ -109,24 +107,26 @@ GgpRsyncClient::GgpRsyncClient(const Options& options, PathFilter path_filter, kForwardPortFirst, kForwardPortLast, &process_factory_, &remote_util_), printer_(options.quiet, Util::IsTTY() && !options.json), - progress_(&printer_, options.verbosity, options.json) {} + progress_(&printer_, options.verbosity, options.json) { + if (!options_.ssh_command.empty()) { + remote_util_.SetSshCommand(options_.ssh_command); + } + if (!options_.scp_command.empty()) { + remote_util_.SetScpCommand(options_.scp_command); + } +} -GgpRsyncClient::~GgpRsyncClient() { +CdcRsyncClient::~CdcRsyncClient() { message_pump_.StopMessagePump(); socket_.Disconnect(); } -absl::Status GgpRsyncClient::Run() { - absl::Status status = remote_util_.GetInitStatus(); - if (!status.ok()) { - return WrapStatus(status, "Failed to initialize critical components"); - } - +absl::Status CdcRsyncClient::Run() { // Initialize |remote_util_|. - remote_util_.SetIpAndPort(options_.ip, options_.port); + remote_util_.SetUserHostAndPort(user_host_, options_.port); // Start the server process. - status = StartServer(); + absl::Status status = StartServer(); if (HasTag(status, Tag::kDeployServer)) { // Gamelet components are not deployed or out-dated. Deploy and retry. status = DeployServer(); @@ -166,7 +166,7 @@ absl::Status GgpRsyncClient::Run() { return status; } -absl::Status GgpRsyncClient::StartServer() { +absl::Status CdcRsyncClient::StartServer() { assert(!server_process_); // Components are expected to reside in the same dir as the executable. @@ -187,8 +187,8 @@ absl::Status GgpRsyncClient::StartServer() { std::string component_args = GameletComponent::ToCommandLineArgs(components); // Find available local and remote ports for port forwarding. - absl::StatusOr port_res = - port_manager_.ReservePort(options_.connection_timeout_sec); + absl::StatusOr port_res = port_manager_.ReservePort( + /*check_remote=*/false, /*remote_timeout_sec unused*/ 0); constexpr char kErrorMsg[] = "Failed to find available port"; if (absl::IsDeadlineExceeded(port_res.status())) { // Server didn't respond in time. @@ -205,9 +205,11 @@ absl::Status GgpRsyncClient::StartServer() { std::string(kRemoteToolsBinDir) + kGgpServerFilename; // Test existence manually to prevent misleading bash output message // "bash: .../cdc_rsync_server: No such file or directory". - std::string remote_command = absl::StrFormat( - "if [ ! -f %s ]; then exit %i; fi; %s %i %s", remote_server_path, - kExitCodeNotFound, remote_server_path, port, component_args); + // Also create the bin dir because otherwise scp below might fail. + std::string remote_command = + absl::StrFormat("mkdir -p %s; if [ ! -f %s ]; then exit %i; fi; %s %i %s", + kRemoteToolsBinDir, remote_server_path, kExitCodeNotFound, + remote_server_path, port, component_args); ProcessStartInfo start_info = remote_util_.BuildProcessStartInfoForSshPortForwardAndCommand( port, port, false, remote_command); @@ -225,16 +227,25 @@ absl::Status GgpRsyncClient::StartServer() { } // Wait until the server process is listening. - auto detect_listening = [is_listening = &is_server_listening_]() -> bool { - return *is_listening; + Stopwatch timeout_timer; + bool is_timeout = false; + auto detect_listening_or_timeout = [is_listening = &is_server_listening_, + timeout = options_.connection_timeout_sec, + &timeout_timer, &is_timeout]() -> bool { + is_timeout = timeout_timer.ElapsedSeconds() > timeout; + return *is_listening || is_timeout; }; - status = process->RunUntil(detect_listening); + status = process->RunUntil(detect_listening_or_timeout); if (!status.ok()) { // Some internal process error. Note that this does NOT mean that // cdc_rsync_server does not exist. In that case, the ssh process exits with // code 127. return status; } + if (is_timeout) { + return SetTag(absl::DeadlineExceededError("Timeout while starting server"), + Tag::kConnectionTimeout); + } if (process->HasExited()) { // Don't re-deploy for code > kServerExitCodeOutOfDate, which means that the @@ -263,7 +274,7 @@ absl::Status GgpRsyncClient::StartServer() { return absl::OkStatus(); } -absl::Status GgpRsyncClient::StopServer() { +absl::Status CdcRsyncClient::StopServer() { assert(server_process_); // Close socket. @@ -282,7 +293,7 @@ absl::Status GgpRsyncClient::StopServer() { return absl::OkStatus(); } -absl::Status GgpRsyncClient::HandleServerOutput(const char* data) { +absl::Status CdcRsyncClient::HandleServerOutput(const char* data) { // Note: This is called from a background thread! // Handle server error messages. Unfortunately, if the server prints to @@ -319,7 +330,7 @@ absl::Status GgpRsyncClient::HandleServerOutput(const char* data) { return absl::OkStatus(); } -absl::Status GgpRsyncClient::Sync() { +absl::Status CdcRsyncClient::Sync() { absl::Status status = SendOptions(); if (!status.ok()) { return WrapStatus(status, "Failed to send options to server"); @@ -377,7 +388,7 @@ absl::Status GgpRsyncClient::Sync() { return status; } -absl::Status GgpRsyncClient::DeployServer() { +absl::Status CdcRsyncClient::DeployServer() { assert(!server_process_); std::string exe_dir; @@ -409,34 +420,26 @@ absl::Status GgpRsyncClient::DeployServer() { return WrapStatus(status, "Failed to copy cdc_rsync_server to instance"); } - // Make cdc_rsync_server executable. - status = remote_util_.Chmod("a+x", remoteServerTmpPath); + // Do 3 things in one SSH command, to save time: + // - Make the old cdc_rsync_server writable (if it exists). + // - Make the new cdc_rsync_server executable. + // - Replace the old cdc_rsync_server by the new one. + std::string old_path = RemoteUtil::EscapeForWindows( + std::string(kRemoteToolsBinDir) + kGgpServerFilename); + std::string new_path = RemoteUtil::EscapeForWindows(remoteServerTmpPath); + std::string replace_cmd = absl::StrFormat( + " ([ ! -f %s ] || chmod u+w %s) && chmod a+x %s && mv %s %s", old_path, + old_path, new_path, new_path, old_path); + status = remote_util_.Run(replace_cmd, "chmod && chmod && mv"); if (!status.ok()) { return WrapStatus(status, - "Failed to set executable flag on cdc_rsync_server"); - } - - // Make old file writable. Mv might fail to overwrite it, e.g. if someone made - // it read-only. - std::string remoteServerPath = - std::string(kRemoteToolsBinDir) + kGgpServerFilename; - status = remote_util_.Chmod("u+w", remoteServerPath, /*quiet=*/true); - if (!status.ok()) { - LOG_DEBUG("chmod u+w %s failed (expected if file does not exist): %s", - remoteServerPath, status.ToString()); - } - - // Replace old file by new file. - status = remote_util_.Mv(remoteServerTmpPath, remoteServerPath); - if (!status.ok()) { - return WrapStatus(status, "Failed to replace '%s' by '%s'", - remoteServerPath, remoteServerTmpPath); + "Failed to replace old cdc_rsync_server by new one"); } return absl::OkStatus(); } -absl::Status GgpRsyncClient::SendOptions() { +absl::Status CdcRsyncClient::SendOptions() { LOG_INFO("Sending options"); SetOptionsRequest request; @@ -448,7 +451,7 @@ absl::Status GgpRsyncClient::SendOptions() { request.set_compress(options_.compress); request.set_relative(options_.relative); - for (const PathFilter::Rule& rule : path_filter_.GetRules()) { + for (const PathFilter::Rule& rule : options_.filter.GetRules()) { SetOptionsRequest::FilterRule* filter_rule = request.add_filter_rules(); filter_rule->set_type(ToProtoType(rule.type)); filter_rule->set_pattern(rule.pattern); @@ -457,7 +460,7 @@ absl::Status GgpRsyncClient::SendOptions() { request.set_checksum(options_.checksum); request.set_dry_run(options_.dry_run); request.set_existing(options_.existing); - if (options_.copy_dest) { + if (!options_.copy_dest.empty()) { request.set_copy_dest(options_.copy_dest); } @@ -470,13 +473,13 @@ absl::Status GgpRsyncClient::SendOptions() { return absl::OkStatus(); } -absl::Status GgpRsyncClient::FindAndSendAllSourceFiles() { +absl::Status CdcRsyncClient::FindAndSendAllSourceFiles() { LOG_INFO("Finding and sending all sources files"); Stopwatch stopwatch; - FileFinderAndSender file_finder(&path_filter_, &message_pump_, &progress_, - sources_dir_, options_.recursive, + FileFinderAndSender file_finder(&options_.filter, &message_pump_, &progress_, + options_.sources_dir, options_.recursive, options_.relative); progress_.StartFindFiles(); @@ -497,7 +500,7 @@ absl::Status GgpRsyncClient::FindAndSendAllSourceFiles() { return absl::OkStatus(); } -absl::Status GgpRsyncClient::ReceiveFileStats() { +absl::Status CdcRsyncClient::ReceiveFileStats() { LOG_INFO("Receiving file stats"); SendFileStatsResponse response; @@ -517,7 +520,7 @@ absl::Status GgpRsyncClient::ReceiveFileStats() { return absl::OkStatus(); } -absl::Status GgpRsyncClient::ReceiveDeletedFiles() { +absl::Status CdcRsyncClient::ReceiveDeletedFiles() { LOG_INFO("Receiving path of deleted files"); std::string current_directory; @@ -548,7 +551,7 @@ absl::Status GgpRsyncClient::ReceiveDeletedFiles() { return absl::OkStatus(); } -absl::Status GgpRsyncClient::ReceiveFileIndices( +absl::Status CdcRsyncClient::ReceiveFileIndices( const char* file_type, std::vector* file_indices) { LOG_INFO("Receiving indices of %s files", file_type); @@ -582,7 +585,7 @@ absl::Status GgpRsyncClient::ReceiveFileIndices( return absl::OkStatus(); } -absl::Status GgpRsyncClient::SendMissingFiles() { +absl::Status CdcRsyncClient::SendMissingFiles() { if (missing_file_indices_.empty()) { return absl::OkStatus(); } @@ -653,7 +656,7 @@ absl::Status GgpRsyncClient::SendMissingFiles() { return absl::OkStatus(); } -absl::Status GgpRsyncClient::ReceiveSignaturesAndSendDelta() { +absl::Status CdcRsyncClient::ReceiveSignaturesAndSendDelta() { if (changed_file_indices_.empty()) { return absl::OkStatus(); } @@ -731,7 +734,7 @@ absl::Status GgpRsyncClient::ReceiveSignaturesAndSendDelta() { return absl::OkStatus(); } -absl::Status GgpRsyncClient::StartCompressionStream() { +absl::Status CdcRsyncClient::StartCompressionStream() { assert(!compression_stream_); // Notify server that data is compressed from now on. @@ -762,7 +765,7 @@ absl::Status GgpRsyncClient::StartCompressionStream() { return absl::OkStatus(); } -absl::Status GgpRsyncClient::StopCompressionStream() { +absl::Status CdcRsyncClient::StopCompressionStream() { assert(compression_stream_); // Finish writing to |compression_process_|'s stdin and change back to diff --git a/cdc_rsync/cdc_rsync_client.h b/cdc_rsync/cdc_rsync_client.h index 20203ae..e9680d2 100644 --- a/cdc_rsync/cdc_rsync_client.h +++ b/cdc_rsync/cdc_rsync_client.h @@ -22,7 +22,6 @@ #include "absl/status/status.h" #include "cdc_rsync/base/message_pump.h" -#include "cdc_rsync/cdc_rsync.h" #include "cdc_rsync/client_socket.h" #include "cdc_rsync/progress_tracker.h" #include "common/path_filter.h" @@ -34,13 +33,38 @@ namespace cdc_ft { class Process; class ZstdStream; -class GgpRsyncClient { +class CdcRsyncClient { public: - GgpRsyncClient(const Options& options, PathFilter filter, - std::string sources_dir, std::vector sources, - std::string destination); + struct Options { + int port = RemoteUtil::kDefaultSshPort; + bool delete_ = false; + bool recursive = false; + int verbosity = 0; + bool quiet = false; + bool whole_file = false; + bool relative = false; + bool compress = false; + bool checksum = false; + bool dry_run = false; + bool existing = false; + bool json = false; + std::string copy_dest; + int compress_level = 6; + int connection_timeout_sec = 10; + std::string ssh_command; + std::string scp_command; + std::string sources_dir; // Base dir for files loaded for --files-from. + PathFilter filter; - ~GgpRsyncClient(); + // Compression level 0 is invalid. + static constexpr int kMinCompressLevel = -5; + static constexpr int kMaxCompressLevel = 22; + }; + + CdcRsyncClient(const Options& options, std::vector sources, + std::string user_host, std::string destination); + + ~CdcRsyncClient(); // Deploys the server if necessary, starts it and runs the rsync procedure. absl::Status Run(); @@ -93,11 +117,9 @@ class GgpRsyncClient { absl::Status StopCompressionStream(); Options options_; - PathFilter path_filter_; - const std::string sources_dir_; std::vector sources_; + const std::string user_host_; const std::string destination_; - WinProcessFactory process_factory_; RemoteUtil remote_util_; PortManager port_manager_; diff --git a/cdc_rsync/dllmain.cc b/cdc_rsync/dllmain.cc deleted file mode 100644 index 337cc93..0000000 --- a/cdc_rsync/dllmain.cc +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright 2022 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#define WIN32_LEAN_AND_MEAN -#include - -BOOL APIENTRY DllMain(HMODULE /* hModule */, DWORD ul_reason_for_call, - LPVOID /* lpReserved */ -) { - switch (ul_reason_for_call) { - case DLL_PROCESS_ATTACH: - case DLL_THREAD_ATTACH: - case DLL_THREAD_DETACH: - case DLL_PROCESS_DETACH: - break; - } - return TRUE; -} diff --git a/cdc_rsync/error_messages.h b/cdc_rsync/error_messages.h deleted file mode 100644 index 6268a23..0000000 --- a/cdc_rsync/error_messages.h +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef CDC_RSYNC_ERROR_MESSAGES_H_ -#define CDC_RSYNC_ERROR_MESSAGES_H_ - -namespace cdc_ft { - -// Server connection timed out. SSH probably stale. -constexpr char kMsgFmtConnectionTimeout[] = - "Server connection timed out. Please re-run 'ggp ssh init' and verify that " - "the IP '%s' and the port '%i' are correct."; - -// Server connection timed out and IP was not passed in. Probably network error. -constexpr char kMsgConnectionTimeoutWithIp[] = - "Server connection timed out. Please check your network connection."; - -// Receiving pipe end was shut down unexpectedly. -constexpr char kMsgConnectionLost[] = - "The connection to the instance was shut down unexpectedly."; - -// Binding to the port failed. -constexpr char kMsgAddressInUse[] = - "Failed to establish a connection to the instance. All ports are already " - "in use. This can happen if another instance of this command is running. " - "Currently, only 10 simultaneous connections are supported."; - -// Deployment failed even though gamelet components were copied successfully. -constexpr char kMsgDeployFailed[] = - "Failed to deploy the instance components for unknown reasons. " - "Please report this issue."; - -// Picking an instance is not allowed in quiet mode. -constexpr char kMsgInstancePickerNotAvailableInQuietMode[] = - "Multiple gamelet instances are reserved, but the instance picker is not " - "available in quiet mode. Please specify --instance or remove -q resp. " - "--quiet."; - -} // namespace cdc_ft - -#endif // CDC_RSYNC_ERROR_MESSAGES_H_ diff --git a/cdc_rsync/main.cc b/cdc_rsync/main.cc new file mode 100644 index 0000000..3dd475e --- /dev/null +++ b/cdc_rsync/main.cc @@ -0,0 +1,147 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#define WIN32_LEAN_AND_MEAN +#include + +#include +#include + +#include "cdc_rsync/cdc_rsync_client.h" +#include "cdc_rsync/params.h" +#include "common/log.h" +#include "common/status.h" +#include "common/util.h" + +namespace { + +enum class ReturnCode { + // No error. Will match the tool's exit code, so OK must be 0. + kOk = 0, + + // Generic error. + kGenericError = 1, + + // Server connection timed out. + kConnectionTimeout = 2, + + // Connection to the server was shut down unexpectedly. + kConnectionLost = 3, + + // Binding to the forward port failed, probably because there's another + // instance of cdc_rsync running. + kAddressInUse = 4, + + // Server deployment failed. This should be rare, it means that the server + // components were successfully copied, but the up-to-date check still fails. + kDeployFailed = 5, +}; + +ReturnCode TagToMessage(cdc_ft::Tag tag, + const cdc_ft::params::Parameters& params, + std::string* msg) { + msg->clear(); + switch (tag) { + case cdc_ft::Tag::kSocketEof: + // Receiving pipe end was shut down unexpectedly. + *msg = "The connection to the instance was shut down unexpectedly."; + return ReturnCode::kConnectionLost; + + case cdc_ft::Tag::kAddressInUse: + *msg = + "Failed to establish a connection to the instance. All ports are " + "already in use. This can happen if another instance of this command " + "is running. Currently, only 10 simultaneous connections are " + "supported."; + return ReturnCode::kAddressInUse; + + case cdc_ft::Tag::kDeployServer: + *msg = + "Failed to deploy the instance components for unknown reasons. " + "Please report this issue."; + return ReturnCode::kDeployFailed; + + case cdc_ft::Tag::kConnectionTimeout: + // Server connection timed out. SSH probably stale. + *msg = absl::StrFormat( + "Server connection timed out. Verify that host '%s' and port '%i' " + "are correct, or specify a larger timeout with --contimeout.", + params.user_host, params.options.port); + return ReturnCode::kConnectionTimeout; + + case cdc_ft::Tag::kCount: + return ReturnCode::kGenericError; + } + + // Should not happen (TM). Will fall back to status message in this case. + return ReturnCode::kGenericError; +} + +} // namespace + +int wmain(int argc, wchar_t* argv[]) { + // Convert args from wide to UTF8 strings. + std::vector utf8_str_args; + utf8_str_args.reserve(argc); + for (int i = 0; i < argc; i++) { + utf8_str_args.push_back(cdc_ft::Util::WideToUtf8Str(argv[i])); + } + + // Convert args from UTF8 strings to UTF8 c-strings. + std::vector utf8_args; + utf8_args.reserve(argc); + for (const auto& utf8_str_arg : utf8_str_args) { + utf8_args.push_back(utf8_str_arg.c_str()); + } + + // Read parameters from the environment and the command line. + cdc_ft::params::Parameters parameters; + if (!cdc_ft::params::Parse(argc, utf8_args.data(), ¶meters)) { + return 1; + } + + // Initialize logging. + cdc_ft::LogLevel log_level = + cdc_ft::Log::VerbosityToLogLevel(parameters.options.verbosity); + cdc_ft::Log::Initialize(std::make_unique(log_level)); + + // Run rsync. + cdc_ft::CdcRsyncClient client(parameters.options, parameters.sources, + parameters.user_host, parameters.destination); + absl::Status status = client.Run(); + if (status.ok()) { + return static_cast(ReturnCode::kOk); + } + + // Get an error message from the tag associated with the status. + std::string error_message; + ReturnCode code = ReturnCode::kGenericError; + absl::optional tag = cdc_ft::GetTag(status); + if (tag.has_value()) { + code = TagToMessage(tag.value(), parameters, &error_message); + } + + // Fall back to status message if there was no tag. + if (error_message.empty()) { + error_message = status.message(); + } else if (parameters.options.verbosity >= 2) { + // In verbose mode, log the status as well, so nothing gets lost. + LOG_ERROR("%s", status.ToString()); + } + + if (!error_message.empty()) { + fprintf(stderr, "Error: %s\n", error_message.c_str()); + } + return static_cast(code); +} diff --git a/cdc_rsync_cli/params.cc b/cdc_rsync/params.cc similarity index 74% rename from cdc_rsync_cli/params.cc rename to cdc_rsync/params.cc index 665b14a..d98597a 100644 --- a/cdc_rsync_cli/params.cc +++ b/cdc_rsync/params.cc @@ -12,12 +12,13 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "cdc_rsync_cli/params.h" +#include "cdc_rsync/params.h" #include #include "absl/status/status.h" #include "absl/strings/str_format.h" +#include "absl/strings/str_split.h" #include "common/path.h" #include "lib/zstd.h" @@ -25,6 +26,8 @@ namespace cdc_ft { namespace params { namespace { +using Options = CdcRsyncClient::Options; + template void PrintError(const absl::FormatSpec& format, Args... args) { std::cerr << "Error: " << absl::StrFormat(format, args...) << std::endl; @@ -39,11 +42,13 @@ Synchronizes local files and files on a gamelet. Matching files are skipped. For partially matching files only the deltas are transferred. Usage: - cdc_rsync [options] source [source]... destination + cdc_rsync [options] source [source]... [user@]host:destination Parameters: - source Local file or folder to be copied - destination Destination folder on the gamelet + source Local file or directory to be copied + user Remote SSH user name + host Remote host or IP address + destination Remote destination directory Options: --ip string Gamelet IP. Required. @@ -54,7 +59,7 @@ Options: --json Print JSON progress -n, --dry-run Perform a trial run with no changes made -r, --recursive Recurse into directories - --delete Delete extraneous files from destination folder + --delete Delete extraneous files from destination directory -z, --compress Compress file data during the transfer --compress-level num Explicitly set compression level (default: 6) -c, --checksum Skip files based on checksum, not mod-time & size @@ -68,13 +73,29 @@ Options: -R, --relative Use relative path names --existing Skip creating new files on instance --copy-dest dir Use files from dir as sync base if files are missing - from destination folder + --ssh-command Path and arguments of SSH command to use, e.g. + C:\path\to\ssh.exe -F config -i id_rsa -oStrictHostKeyChecking=yes -oUserKnownHostsFile="""known_hosts""" + Can also be specified by the CDC_SSH_COMMAND environment variable. + --scp-command Path and arguments of SSH command to use, e.g. + C:\path\to\scp.exe -F config -i id_rsa -oStrictHostKeyChecking=yes -oUserKnownHostsFile="""known_hosts""" + Can also be specified by the CDC_SCP_COMMAND environment variable. -h --help Help for cdc_rsync )"; +constexpr char kSshCommandEnvVar[] = "CDC_SSH_COMMAND"; +constexpr char kScpCommandEnvVar[] = "CDC_SCP_COMMAND"; + +// Populates some parameters from environment variables. +void PopulateFromEnvVars(Parameters* parameters) { + path::GetEnv(kSshCommandEnvVar, ¶meters->options.ssh_command) + .IgnoreError(); + path::GetEnv(kScpCommandEnvVar, ¶meters->options.scp_command) + .IgnoreError(); +} + // Handles the --exclude-from and --include-from options. OptionResult HandleFilterRuleFile(const std::string& option_name, - const char* path, FilterRule::Type type, + const char* path, PathFilter::Rule::Type type, Parameters* params) { if (!path) { PrintError("Option '%s' needs a value", option_name); @@ -92,7 +113,7 @@ OptionResult HandleFilterRuleFile(const std::string& option_name, } for (std::string& pattern : patterns) { - params->filter_rules.emplace_back(type, std::move(pattern)); + params->options.filter.AddRule(type, std::move(pattern)); } return OptionResult::kConsumedKeyValue; } @@ -143,11 +164,6 @@ bool LoadFilesFrom(const std::string& files_from, OptionResult HandleParameter(const std::string& key, const char* value, Parameters* params, bool* help) { - if (key == "ip") { - params->options.ip = value; - return OptionResult::kConsumedKeyValue; - } - if (key == "port") { if (value) { params->options.port = atoi(value); @@ -181,27 +197,29 @@ OptionResult HandleParameter(const std::string& key, const char* value, } if (key == "include") { - params->filter_rules.emplace_back(FilterRule::Type::kInclude, value); + params->options.filter.AddRule(PathFilter::Rule::Type::kInclude, value); return OptionResult::kConsumedKeyValue; } if (key == "include-from") { - return HandleFilterRuleFile(key, value, FilterRule::Type::kInclude, params); + return HandleFilterRuleFile(key, value, PathFilter::Rule::Type::kInclude, + params); } if (key == "exclude") { - params->filter_rules.emplace_back(FilterRule::Type::kExclude, value); + params->options.filter.AddRule(PathFilter::Rule::Type::kExclude, value); return OptionResult::kConsumedKeyValue; } if (key == "exclude-from") { - return HandleFilterRuleFile(key, value, FilterRule::Type::kExclude, params); + return HandleFilterRuleFile(key, value, PathFilter::Rule::Type::kExclude, + params); } if (key == "files-from") { // Implies -R. params->options.relative = true; - params->files_from = value; + params->files_from = value ? value : std::string(); return OptionResult::kConsumedKeyValue; } @@ -250,7 +268,7 @@ OptionResult HandleParameter(const std::string& key, const char* value, } if (key == "copy-dest") { - params->options.copy_dest = value; + params->options.copy_dest = value ? value : std::string(); return OptionResult::kConsumedKeyValue; } @@ -259,13 +277,23 @@ OptionResult HandleParameter(const std::string& key, const char* value, return OptionResult::kConsumedKey; } + if (key == "ssh-command") { + params->options.ssh_command = value ? value : std::string(); + return OptionResult::kConsumedKeyValue; + } + + if (key == "scp-command") { + params->options.scp_command = value ? value : std::string(); + return OptionResult::kConsumedKeyValue; + } + PrintError("Unknown option: '%s'", key); return OptionResult::kError; } -bool CheckParameters(const Parameters& params, bool help) { +bool ValidateParameters(const Parameters& params, bool help) { if (help) { - printf("%s", kHelpText); + std::cout << kHelpText; return false; } @@ -274,13 +302,7 @@ bool CheckParameters(const Parameters& params, bool help) { return false; } - if (!params.options.ip || params.options.ip[0] == '\0') { - PrintError("--ip must specify a valid IP address"); - return false; - } - - if (!params.options.port || params.options.port <= 0 || - params.options.port > UINT16_MAX) { + if (params.options.port <= 0 || params.options.port > UINT16_MAX) { PrintError("--port must specify a valid port"); return false; } @@ -301,9 +323,10 @@ bool CheckParameters(const Parameters& params, bool help) { // Warn that any include rules not followed by an exclude rule are pointless // as the files would be included, anyway. - for (int n = static_cast(params.filter_rules.size()) - 1; n >= 0; --n) { - const Parameters::FilterRule& rule = params.filter_rules[n]; - if (rule.type == FilterRule::Type::kExclude) { + const std::vector& rules = params.options.filter.GetRules(); + for (int n = static_cast(rules.size()) - 1; n >= 0; --n) { + const PathFilter::Rule& rule = rules[n]; + if (rule.type == PathFilter::Rule::Type::kExclude) { break; } std::cout << "Warning: Include pattern '" << rule.pattern @@ -311,6 +334,31 @@ bool CheckParameters(const Parameters& params, bool help) { << std::endl; } + if (params.sources.empty() && params.destination.empty()) { + PrintError("Missing source and destination"); + return false; + } + + if (params.destination.empty()) { + PrintError("Missing destination"); + return false; + } + + if (params.sources.empty()) { + // If one arg was passed on the command line, it is not clear whether it + // was supposed to be a source or destination. + PrintError("Missing source or destination"); + return false; + } + + if (params.user_host.empty()) { + PrintError( + "No remote host specified in destination '%s'. " + "Expected [user@]host:destination.", + params.destination); + return false; + } + return true; } @@ -335,6 +383,25 @@ bool CheckOptionResult(OptionResult result, const std::string& name, return true; } +// Removes the user/host part of |destination| and puts it into |user_host|, +// e.g. if |destination| is initially "user@foo.com:~/file", it is "~/file" +// afterward and |user_host| is |user@foo.com|. Does not touch Windows drives, +// e.g. C:\foo. +void PopUserHost(std::string* destination, std::string* user_host) { + std::vector parts = + absl::StrSplit(*destination, absl::MaxSplits(':', 1)); + if (parts.size() < 2) return; + + // Don't mistake the C part of C:\foo as user/host. + if (parts[0].size() == 1 && toupper(parts[0][0]) >= 'A' && + toupper(parts[0][0]) <= 'Z') { + return; + } + + *user_host = parts[0]; + *destination = parts[1]; +} + } // namespace const char* HelpText() { return kHelpText; } @@ -349,6 +416,9 @@ bool Parse(int argc, const char* const* argv, Parameters* parameters) { return false; } + // Before applying args, populate parameters from env vars. + PopulateFromEnvVars(parameters); + bool help = false; for (int index = 1; index < argc; ++index) { // Handle '--key [value]' and '--key=value' options. @@ -404,34 +474,15 @@ bool Parse(int argc, const char* const* argv, Parameters* parameters) { // Load files-from file (can't do it when --files-from is handled since not // all sources might have been read at that point. - if (parameters->files_from && + if (!parameters->files_from.empty() && !LoadFilesFrom(parameters->files_from, ¶meters->sources, - ¶meters->sources_dir)) { + ¶meters->options.sources_dir)) { return false; } - if (!CheckParameters(*parameters, help)) { - return false; - } + PopUserHost(¶meters->destination, ¶meters->user_host); - if (parameters->sources.empty() && parameters->destination.empty()) { - PrintError("Missing source and destination"); - return false; - } - - if (parameters->destination.empty()) { - PrintError("Missing destination"); - return false; - } - - if (parameters->sources.empty()) { - // If one arg was passed on the command line, it is not clear whether it - // was supposed to be a source or destination. Try to infer that, e.g. - // cdc_rsync *.txt -> Missing destination - // cdc_rsync /mnt/developer -> Missing source - bool missing_src = parameters->destination[0] == '/'; - - PrintError("Missing %s", missing_src ? "source" : "destination"); + if (!ValidateParameters(*parameters, help)) { return false; } diff --git a/cdc_rsync_cli/params.h b/cdc_rsync/params.h similarity index 64% rename from cdc_rsync_cli/params.h rename to cdc_rsync/params.h index 97f0ec0..e278a21 100644 --- a/cdc_rsync_cli/params.h +++ b/cdc_rsync/params.h @@ -14,34 +14,24 @@ * limitations under the License. */ -#ifndef CDC_RSYNC_CLI_PARAMS_H_ -#define CDC_RSYNC_CLI_PARAMS_H_ +#ifndef CDC_RSYNC_PARAMS_H_ +#define CDC_RSYNC_PARAMS_H_ #include #include -#include "cdc_rsync/cdc_rsync.h" +#include "cdc_rsync/cdc_rsync_client.h" namespace cdc_ft { namespace params { // All cdc_rsync command line parameters. struct Parameters { - // Copy of cdc_ft::FilterRule with std::string instead of const char*. - struct FilterRule { - using Type = ::cdc_ft::FilterRule::Type; - FilterRule(Type type, std::string pattern) - : type(type), pattern(std::move(pattern)) {} - Type type; - std::string pattern; - }; - - Options options; - std::vector filter_rules; + CdcRsyncClient::Options options; std::vector sources; + std::string user_host; std::string destination; - const char* files_from = nullptr; - std::string sources_dir; // Base directory for files loaded for --files-from. + std::string files_from; }; // Parses sources, destination and options from the command line args. @@ -51,4 +41,4 @@ bool Parse(int argc, const char* const* argv, Parameters* parameters); } // namespace params } // namespace cdc_ft -#endif // CDC_RSYNC_CLI_PARAMS_H_ +#endif // CDC_RSYNC_PARAMS_H_ diff --git a/cdc_rsync_cli/params_test.cc b/cdc_rsync/params_test.cc similarity index 55% rename from cdc_rsync_cli/params_test.cc rename to cdc_rsync/params_test.cc index 1315427..ea3b8c0 100644 --- a/cdc_rsync_cli/params_test.cc +++ b/cdc_rsync/params_test.cc @@ -12,11 +12,12 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "cdc_rsync_cli/params.h" +#include "cdc_rsync/params.h" #include "absl/strings/match.h" #include "common/log.h" #include "common/path.h" +#include "common/status_test_macros.h" #include "common/test_main.h" #include "gtest/gtest.h" @@ -24,6 +25,13 @@ namespace cdc_ft { namespace params { namespace { +using Options = CdcRsyncClient::Options; + +constexpr char kSrc[] = "source"; +constexpr char kUserHostDst[] = "user@host:destination"; +constexpr char kUserHost[] = "user@host"; +constexpr char kDst[] = "destination"; + class TestLog : public Log { public: explicit TestLog() : Log(LogLevel::kInfo) {} @@ -44,9 +52,15 @@ std::string NeedsValueError(const char* option_name) { class ParamsTest : public ::testing::Test { public: - void SetUp() override { prev_stderr_ = std::cerr.rdbuf(errors_.rdbuf()); } + void SetUp() override { + prev_stdout_ = std::cout.rdbuf(output_.rdbuf()); + prev_stderr_ = std::cerr.rdbuf(errors_.rdbuf()); + } - void TearDown() override { std::cerr.rdbuf(prev_stderr_); } + void TearDown() override { + std::cout.rdbuf(prev_stdout_); + std::cerr.rdbuf(prev_stderr_); + } protected: void ExpectNoError() const { @@ -54,6 +68,12 @@ class ParamsTest : public ::testing::Test { << "Expected empty stderr but got\n'" << errors_.str() << "'"; } + void ExpectOutput(const std::string& expected) const { + EXPECT_TRUE(absl::StrContains(output_.str(), expected)) + << "Expected stdout to contain '" << expected << "' but got\n'" + << output_.str() << "'"; + } + void ExpectError(const std::string& expected) const { EXPECT_TRUE(absl::StrContains(errors_.str(), expected)) << "Expected stderr to contain '" << expected << "' but got\n'" @@ -68,16 +88,16 @@ class ParamsTest : public ::testing::Test { path::Join(base_dir_, "empty_source_files.txt"); Parameters parameters_; + std::stringstream output_; std::stringstream errors_; + std::streambuf* prev_stdout_; std::streambuf* prev_stderr_; }; TEST_F(ParamsTest, ParseSucceedsDefaults) { - const char* argv[] = {"cdc_rsync.exe", "--ip=1.2.3.4", "--port=1234", - "source", "destination", NULL}; + const char* argv[] = {"cdc_rsync.exe", kSrc, kUserHostDst, NULL}; EXPECT_TRUE(Parse(static_cast(std::size(argv)) - 1, argv, ¶meters_)); - EXPECT_STREQ("1.2.3.4", parameters_.options.ip); - EXPECT_EQ(1234, parameters_.options.port); + EXPECT_EQ(RemoteUtil::kDefaultSshPort, parameters_.options.port); EXPECT_FALSE(parameters_.options.delete_); EXPECT_FALSE(parameters_.options.recursive); EXPECT_EQ(0, parameters_.options.verbosity); @@ -86,19 +106,19 @@ TEST_F(ParamsTest, ParseSucceedsDefaults) { EXPECT_FALSE(parameters_.options.compress); EXPECT_FALSE(parameters_.options.checksum); EXPECT_FALSE(parameters_.options.dry_run); - EXPECT_EQ(parameters_.options.copy_dest, nullptr); + EXPECT_TRUE(parameters_.options.copy_dest.empty()); EXPECT_EQ(6, parameters_.options.compress_level); EXPECT_EQ(10, parameters_.options.connection_timeout_sec); EXPECT_EQ(1, parameters_.sources.size()); - EXPECT_EQ(parameters_.sources[0], "source"); - EXPECT_EQ(parameters_.destination, "destination"); + EXPECT_EQ(parameters_.sources[0], kSrc); + EXPECT_EQ(parameters_.user_host, kUserHost); + EXPECT_EQ(parameters_.destination, kDst); ExpectNoError(); } TEST_F(ParamsTest, ParseSucceedsWithOptionFromTwoArguments) { const char* argv[] = { - "cdc_rsync.exe", "--ip=1.2.3.4", "--port=1234", "--compress-level", "2", - "source", "destination", NULL}; + "cdc_rsync.exe", "--compress-level", "2", kSrc, kUserHostDst, NULL}; EXPECT_TRUE(Parse(static_cast(std::size(argv)) - 1, argv, ¶meters_)); EXPECT_EQ(parameters_.options.compress_level, 2); ExpectNoError(); @@ -106,68 +126,104 @@ TEST_F(ParamsTest, ParseSucceedsWithOptionFromTwoArguments) { TEST_F(ParamsTest, ParseSucceedsWithOptionFromOneArgumentWithEqualityWithValue) { - const char* argv[] = { - "cdc_rsync.exe", "--ip=1.2.3.4", "--port=1234", "--compress-level=2", - "source", "destination", NULL}; + const char* argv[] = {"cdc_rsync.exe", "--compress-level=2", kSrc, + kUserHostDst, NULL}; EXPECT_TRUE(Parse(static_cast(std::size(argv)) - 1, argv, ¶meters_)); ASSERT_EQ(parameters_.sources.size(), 1); EXPECT_EQ(parameters_.options.compress_level, 2); - EXPECT_EQ(parameters_.sources[0], "source"); - EXPECT_EQ(parameters_.destination, "destination"); + EXPECT_EQ(parameters_.sources[0], kSrc); + EXPECT_EQ(parameters_.user_host, kUserHost); + EXPECT_EQ(parameters_.destination, kDst); ExpectNoError(); } TEST_F(ParamsTest, ParseFailsOnCompressLevelEqualsNoValue) { - const char* argv[] = {"cdc_rsync.exe", "--compress-level=", "source", - "destination", NULL}; + const char* argv[] = {"cdc_rsync.exe", "--compress-level=", kSrc, + kUserHostDst, NULL}; EXPECT_FALSE( Parse(static_cast(std::size(argv)) - 1, argv, ¶meters_)); ExpectError(NeedsValueError("compress-level")); } TEST_F(ParamsTest, ParseFailsOnPortEqualsNoValue) { - const char* argv[] = {"cdc_rsync.exe", "--port=", "source", "destination", - NULL}; + const char* argv[] = {"cdc_rsync.exe", "--port=", kSrc, kUserHostDst, NULL}; EXPECT_FALSE( Parse(static_cast(std::size(argv)) - 1, argv, ¶meters_)); ExpectError(NeedsValueError("port")); } TEST_F(ParamsTest, ParseFailsOnContimeoutEqualsNoValue) { - const char* argv[] = {"cdc_rsync.exe", "--contimeout=", "source", - "destination", NULL}; + const char* argv[] = {"cdc_rsync.exe", "--contimeout=", kSrc, kUserHostDst, + NULL}; EXPECT_FALSE( Parse(static_cast(std::size(argv)) - 1, argv, ¶meters_)); ExpectError(NeedsValueError("contimeout")); } -TEST_F(ParamsTest, ParseFailsOnIpEqualsNoValue) { - const char* argv[] = {"cdc_rsync.exe", "--ip=", "source", "destination", +TEST_F(ParamsTest, ParseSucceedsWithSshScpCommands) { + const char* argv[] = {"cdc_rsync.exe", kSrc, + kUserHostDst, "--ssh-command=sshcmd", + "--scp-command=scpcmd", NULL}; + EXPECT_TRUE(Parse(static_cast(std::size(argv)) - 1, argv, ¶meters_)); + EXPECT_EQ(parameters_.options.scp_command, "scpcmd"); + EXPECT_EQ(parameters_.options.ssh_command, "sshcmd"); +} + +TEST_F(ParamsTest, ParseSucceedsWithSshScpCommandsByEnvVars) { + EXPECT_OK(path::SetEnv("CDC_SSH_COMMAND", "sshcmd")); + EXPECT_OK(path::SetEnv("CDC_SCP_COMMAND", "scpcmd")); + const char* argv[] = {"cdc_rsync.exe", kSrc, kUserHostDst, NULL}; + EXPECT_TRUE(Parse(static_cast(std::size(argv)) - 1, argv, ¶meters_)); + EXPECT_EQ(parameters_.options.scp_command, "scpcmd"); + EXPECT_EQ(parameters_.options.ssh_command, "sshcmd"); +} + +TEST_F(ParamsTest, ParseSucceedsWithNoSshCommand) { + const char* argv[] = {"cdc_rsync.exe", kSrc, kUserHostDst, + "--ssh-command=", NULL}; + EXPECT_FALSE( + Parse(static_cast(std::size(argv)) - 1, argv, ¶meters_)); + ExpectError(NeedsValueError("ssh-command")); +} + +TEST_F(ParamsTest, ParseSucceedsWithNoScpCommand) { + const char* argv[] = {"cdc_rsync.exe", kSrc, kUserHostDst, "--scp-command", NULL}; EXPECT_FALSE( Parse(static_cast(std::size(argv)) - 1, argv, ¶meters_)); - ExpectError(NeedsValueError("ip")); + ExpectError(NeedsValueError("scp-command")); +} + +TEST_F(ParamsTest, ParseFailsOnNoUserHost) { + const char* argv[] = {"cdc_rsync.exe", kSrc, kDst, NULL}; + EXPECT_FALSE( + Parse(static_cast(std::size(argv)) - 1, argv, ¶meters_)); + ExpectError("No remote host specified"); +} + +TEST_F(ParamsTest, ParseDoesNotThinkCIsAHost) { + const char* argv[] = {"cdc_rsync.exe", kSrc, "C:\\foo", NULL}; + EXPECT_FALSE( + Parse(static_cast(std::size(argv)) - 1, argv, ¶meters_)); + ExpectError("No remote host specified"); } TEST_F(ParamsTest, ParseWithoutParametersFailsOnMissingSourceAndDestination) { - const char* argv[] = {"cdc_rsync.exe", "--ip=1.2.3.4", "--port=1234", NULL}; + const char* argv[] = {"cdc_rsync.exe", NULL}; EXPECT_FALSE( Parse(static_cast(std::size(argv)) - 1, argv, ¶meters_)); - ExpectError("Missing source"); + ExpectOutput("Usage:"); } TEST_F(ParamsTest, ParseWithSingleParameterFailsOnMissingDestination) { - const char* argv[] = {"cdc_rsync.exe", "--ip=1.2.3.4", "--port=1234", - "source", NULL}; + const char* argv[] = {"cdc_rsync.exe", kSrc, NULL}; EXPECT_FALSE( Parse(static_cast(std::size(argv)) - 1, argv, ¶meters_)); - ExpectError("Missing destination"); + ExpectError("Missing source or destination"); } -TEST_F(ParamsTest, ParseSuccessedsWithMultipleLetterKeyConsumed) { - const char* argv[] = { - "cdc_rsync.exe", "--ip=1.2.3.4", "--port=1234", "-rvqWRzcn", - "source", "destination", NULL}; +TEST_F(ParamsTest, ParseSucceedsWithMultipleLetterKeyConsumed) { + const char* argv[] = {"cdc_rsync.exe", "-rvqWRzcn", kSrc, kUserHostDst, NULL}; EXPECT_TRUE(Parse(static_cast(std::size(argv)) - 1, argv, ¶meters_)); EXPECT_TRUE(parameters_.options.recursive); EXPECT_EQ(parameters_.options.verbosity, 1); @@ -182,17 +238,15 @@ TEST_F(ParamsTest, ParseSuccessedsWithMultipleLetterKeyConsumed) { TEST_F(ParamsTest, ParseFailsOnMultipleLetterKeyConsumedOptionsWithUnsupportedOne) { - const char* argv[] = {"cdc_rsync.exe", "-rvqaWRzcn", "source", "destination", + const char* argv[] = {"cdc_rsync.exe", "-rvqaWRzcn", kSrc, kUserHostDst, NULL}; EXPECT_FALSE( Parse(static_cast(std::size(argv)) - 1, argv, ¶meters_)); ExpectError("Unknown option: 'a'"); } -TEST_F(ParamsTest, ParseSuccessedsWithMultipleLongKeyConsumedOptions) { +TEST_F(ParamsTest, ParseSucceedsWithMultipleLongKeyConsumedOptions) { const char* argv[] = {"cdc_rsync.exe", - "--ip=1.2.3.4", - "--port=1234", "--recursive", "--verbosity", "--quiet", @@ -204,8 +258,8 @@ TEST_F(ParamsTest, ParseSuccessedsWithMultipleLongKeyConsumedOptions) { "--dry-run", "--existing", "--json", - "source", - "destination", + kSrc, + kUserHostDst, NULL}; EXPECT_TRUE(Parse(static_cast(std::size(argv)) - 1, argv, ¶meters_)); EXPECT_TRUE(parameters_.options.recursive); @@ -223,51 +277,42 @@ TEST_F(ParamsTest, ParseSuccessedsWithMultipleLongKeyConsumedOptions) { } TEST_F(ParamsTest, ParseFailsOnUnknownKey) { - const char* argv[] = {"cdc_rsync.exe", "-unknownKey", "source", "destination", + const char* argv[] = {"cdc_rsync.exe", "-unknownKey", kSrc, kUserHostDst, NULL}; EXPECT_FALSE( Parse(static_cast(std::size(argv)) - 1, argv, ¶meters_)); ExpectError("Unknown option: 'u'"); } -TEST_F(ParamsTest, ParseSuccessedsWithSupportedKeyValue) { +TEST_F(ParamsTest, ParseSucceedsWithSupportedKeyValue) { const char* argv[] = { - "cdc_rsync.exe", "--compress-level", "11", "--port=4086", - "--ip=127.0.0.1", "--contimeout", "99", "--copy-dest=dest", - "source", "destination", NULL}; + "cdc_rsync.exe", "--compress-level", "11", "--contimeout", "99", "--port", + "4086", "--copy-dest=dest", kSrc, kUserHostDst, NULL}; EXPECT_TRUE(Parse(static_cast(std::size(argv)) - 1, argv, ¶meters_)); EXPECT_EQ(parameters_.options.compress_level, 11); EXPECT_EQ(parameters_.options.connection_timeout_sec, 99); EXPECT_EQ(parameters_.options.port, 4086); - EXPECT_STREQ(parameters_.options.ip, "127.0.0.1"); - EXPECT_STREQ(parameters_.options.copy_dest, "dest"); + EXPECT_EQ(parameters_.options.copy_dest, "dest"); ExpectNoError(); } -TEST_F(ParamsTest, - ParseSuccessedsWithSupportedKeyValueWithoutEqualityForChars) { - const char* argv[] = {"cdc_rsync.exe", "--port", "4086", "--ip", - "127.0.0.1", "--copy-dest", "dest", "source", - "destination", NULL}; +TEST_F(ParamsTest, ParseSucceedsWithSupportedKeyValueWithoutEqualityForChars) { + const char* argv[] = {"cdc_rsync.exe", "--copy-dest", "dest", kSrc, + kUserHostDst, NULL}; EXPECT_TRUE(Parse(static_cast(std::size(argv)) - 1, argv, ¶meters_)); - EXPECT_EQ(parameters_.options.port, 4086); - EXPECT_STREQ(parameters_.options.ip, "127.0.0.1"); - EXPECT_STREQ(parameters_.options.copy_dest, "dest"); + EXPECT_EQ(parameters_.options.copy_dest, "dest"); ExpectNoError(); } -TEST_F(ParamsTest, ParseFailsOnGameletIpNeedsPort) { - const char* argv[] = {"cdc_rsync.exe", "--ip=127.0.0.1", "source", - "destination", NULL}; +TEST_F(ParamsTest, ParseFailsOnInvalidPort) { + const char* argv[] = {"cdc_rsync.exe", "--port=0", kSrc, kUserHostDst, NULL}; EXPECT_FALSE( Parse(static_cast(std::size(argv)) - 1, argv, ¶meters_)); ExpectError("--port must specify a valid port"); } TEST_F(ParamsTest, ParseFailsOnDeleteNeedsRecursive) { - const char* argv[] = { - "cdc_rsync.exe", "--ip=1.2.3.4", "--port=1234", "--delete", - "source", "destination", NULL}; + const char* argv[] = {"cdc_rsync.exe", "--delete", kSrc, kUserHostDst, NULL}; EXPECT_FALSE( Parse(static_cast(std::size(argv)) - 1, argv, ¶meters_)); ExpectError("--delete does not work without --recursive (-r)"); @@ -281,10 +326,10 @@ TEST_F(ParamsTest, ParseChecksCompressLevel) { for (int n = 0; n < std::size(levels); ++n) { std::string level = "--compress-level=" + std::to_string(levels[n]); - const char* argv[] = {"cdc_rsync.exe", "--ip=1.2.3.4", "--port=1234", - level.c_str(), "source", "destination"}; - EXPECT_TRUE(Parse(static_cast(std::size(argv)) - 1, argv, - ¶meters_) == valid[n]); + const char* argv[] = {"cdc_rsync.exe", level.c_str(), kSrc, kUserHostDst, + NULL}; + EXPECT_EQ(Parse(static_cast(std::size(argv)) - 1, argv, ¶meters_), + valid[n]); if (valid[n]) { ExpectNoError(); } else { @@ -295,94 +340,95 @@ TEST_F(ParamsTest, ParseChecksCompressLevel) { } TEST_F(ParamsTest, ParseFailsOnUnknownKeyValue) { - const char* argv[] = {"cdc_rsync.exe", "--unknownKey=5", "source", - "destination", NULL}; + const char* argv[] = {"cdc_rsync.exe", "--unknownKey=5", kSrc, kUserHostDst, + NULL}; EXPECT_FALSE( Parse(static_cast(std::size(argv)) - 1, argv, ¶meters_)); ExpectError("unknownKey"); } TEST_F(ParamsTest, ParseFailsWithHelpOption) { - const char* argv[] = {"cdc_rsync.exe", "--ip=1.2.3.4", "--port=1234", - "source", "destination", NULL}; + const char* argv[] = {"cdc_rsync.exe", kSrc, kUserHostDst, NULL}; EXPECT_TRUE(Parse(static_cast(std::size(argv)) - 1, argv, ¶meters_)); - const char* argv2[] = { - "cdc_rsync.exe", "--ip=1.2.3.4", "--port=1234", "source", - "destination", "--help", NULL}; + const char* argv2[] = {"cdc_rsync.exe", kSrc, kUserHostDst, "--help", NULL}; EXPECT_FALSE( Parse(static_cast(std::size(argv2)) - 1, argv2, ¶meters_)); ExpectNoError(); - const char* argv3[] = { - "cdc_rsync.exe", "--ip=1.2.3.4", "--port=1234", "source", - "destination", "-h", NULL}; + const char* argv3[] = {"cdc_rsync.exe", kSrc, kUserHostDst, "-h", NULL}; EXPECT_FALSE( Parse(static_cast(std::size(argv3)) - 1, argv3, ¶meters_)); ExpectNoError(); } TEST_F(ParamsTest, ParseSucceedsWithIncludeExclude) { - const char* argv[] = { - "cdc_rsync.exe", "--ip=1.2.3.4", "--port=1234", "--include=*.txt", - "--exclude", "*.dat", "--include", "*.exe", - "source", "destination", NULL}; + const char* argv[] = {"cdc_rsync.exe", + "--include=*.txt", + "--exclude", + "*.dat", + "--include", + "*.exe", + kSrc, + kUserHostDst, + NULL}; EXPECT_TRUE(Parse(static_cast(std::size(argv)) - 1, argv, ¶meters_)); - ASSERT_EQ(parameters_.filter_rules.size(), 3); - ASSERT_EQ(parameters_.filter_rules[0].type, FilterRule::Type::kInclude); - ASSERT_EQ(parameters_.filter_rules[0].pattern, "*.txt"); - ASSERT_EQ(parameters_.filter_rules[1].type, FilterRule::Type::kExclude); - ASSERT_EQ(parameters_.filter_rules[1].pattern, "*.dat"); - ASSERT_EQ(parameters_.filter_rules[2].type, FilterRule::Type::kInclude); - ASSERT_EQ(parameters_.filter_rules[2].pattern, "*.exe"); + const std::vector& rules = + parameters_.options.filter.GetRules(); + ASSERT_EQ(rules.size(), 3); + ASSERT_EQ(rules[0].type, PathFilter::Rule::Type::kInclude); + ASSERT_EQ(rules[0].pattern, "*.txt"); + ASSERT_EQ(rules[1].type, PathFilter::Rule::Type::kExclude); + ASSERT_EQ(rules[1].pattern, "*.dat"); + ASSERT_EQ(rules[2].type, PathFilter::Rule::Type::kInclude); + ASSERT_EQ(rules[2].pattern, "*.exe"); ExpectNoError(); } TEST_F(ParamsTest, FilesFrom_NoFile) { - const char* argv[] = { - "cdc_rsync.exe", "--ip=1.2.3.4", "--port=1234", "source", - "destination", "--files-from", NULL}; + const char* argv[] = {"cdc_rsync.exe", kSrc, kUserHostDst, "--files-from", + NULL}; EXPECT_FALSE( Parse(static_cast(std::size(argv)) - 1, argv, ¶meters_)); ExpectError(NeedsValueError("files-from")); } TEST_F(ParamsTest, FilesFrom_ImpliesRelative) { - const char* argv[] = { - "cdc_rsync.exe", "--ip=1.2.3.4", "--port=1234", "--files-from", - sources_file_.c_str(), base_dir_.c_str(), "destination", NULL}; + const char* argv[] = {"cdc_rsync.exe", "--files-from", + sources_file_.c_str(), base_dir_.c_str(), + kUserHostDst, NULL}; EXPECT_TRUE(Parse(static_cast(std::size(argv)) - 1, argv, ¶meters_)); EXPECT_TRUE(parameters_.options.relative); ExpectNoError(); } TEST_F(ParamsTest, FilesFrom_WithoutSourceArg) { - const char* argv[] = { - "cdc_rsync.exe", "--ip=1.2.3.4", "--port=1234", "--files-from", - sources_file_.c_str(), "destination", NULL}; + const char* argv[] = {"cdc_rsync.exe", "--files-from", sources_file_.c_str(), + kUserHostDst, NULL}; EXPECT_TRUE(Parse(static_cast(std::size(argv)) - 1, argv, ¶meters_)); - EXPECT_TRUE(parameters_.sources_dir.empty()); - EXPECT_EQ(parameters_.destination, "destination"); + EXPECT_TRUE(parameters_.options.sources_dir.empty()); + EXPECT_EQ(parameters_.user_host, kUserHost); + EXPECT_EQ(parameters_.destination, kDst); ExpectNoError(); } TEST_F(ParamsTest, FilesFrom_WithSourceArg) { - const char* argv[] = { - "cdc_rsync.exe", "--ip=1.2.3.4", "--port=1234", "--files-from", - sources_file_.c_str(), base_dir_.c_str(), "destination", NULL}; + const char* argv[] = {"cdc_rsync.exe", "--files-from", + sources_file_.c_str(), base_dir_.c_str(), + kUserHostDst, NULL}; EXPECT_TRUE(Parse(static_cast(std::size(argv)) - 1, argv, ¶meters_)); std::string expected_sources_dir = base_dir_; path::EnsureEndsWithPathSeparator(&expected_sources_dir); - EXPECT_EQ(parameters_.sources_dir, expected_sources_dir); - EXPECT_EQ(parameters_.destination, "destination"); + EXPECT_EQ(parameters_.options.sources_dir, expected_sources_dir); + EXPECT_EQ(parameters_.user_host, kUserHost); + EXPECT_EQ(parameters_.destination, kDst); ExpectNoError(); } TEST_F(ParamsTest, FilesFrom_ParsesFile) { - const char* argv[] = { - "cdc_rsync.exe", "--ip=1.2.3.4", "--port=1234", "--files-from", - sources_file_.c_str(), "destination", NULL}; + const char* argv[] = {"cdc_rsync.exe", "--files-from", sources_file_.c_str(), + kUserHostDst, NULL}; EXPECT_TRUE(Parse(static_cast(std::size(argv)) - 1, argv, ¶meters_)); std::vector expected = {"file1", "file2", "file3"}; @@ -394,13 +440,8 @@ TEST_F(ParamsTest, FilesFrom_ParsesFile) { } TEST_F(ParamsTest, FilesFrom_EmptyFile_WithoutSourceArg) { - const char* argv[] = {"cdc_rsync.exe", - "--ip=1.2.3.4", - "--port=1234", - "--files-from", - empty_sources_file_.c_str(), - "destination", - NULL}; + const char* argv[] = {"cdc_rsync.exe", "--files-from", + empty_sources_file_.c_str(), kUserHostDst, NULL}; EXPECT_FALSE( Parse(static_cast(std::size(argv)) - 1, argv, ¶meters_)); ExpectError(empty_sources_file_); @@ -408,14 +449,9 @@ TEST_F(ParamsTest, FilesFrom_EmptyFile_WithoutSourceArg) { } TEST_F(ParamsTest, FilesFrom_EmptyFile_WithSourceArg) { - const char* argv[] = {"cdc_rsync.exe", - "--ip=1.2.3.4", - "--port=1234", - "--files-from", - empty_sources_file_.c_str(), - base_dir_.c_str(), - "destination", - NULL}; + const char* argv[] = { + "cdc_rsync.exe", "--files-from", empty_sources_file_.c_str(), + base_dir_.c_str(), kUserHostDst, NULL}; EXPECT_FALSE( Parse(static_cast(std::size(argv)) - 1, argv, ¶meters_)); ExpectError(empty_sources_file_); @@ -423,17 +459,16 @@ TEST_F(ParamsTest, FilesFrom_EmptyFile_WithSourceArg) { } TEST_F(ParamsTest, FilesFrom_NoDestination) { - const char* argv[] = {"cdc_rsync.exe", "--ip=1.2.3.4", "--port=1234", - "--files-from", sources_file_.c_str(), NULL}; + const char* argv[] = {"cdc_rsync.exe", "--files-from", sources_file_.c_str(), + NULL}; EXPECT_FALSE( Parse(static_cast(std::size(argv)) - 1, argv, ¶meters_)); ExpectError("Missing destination"); } TEST_F(ParamsTest, IncludeFrom_NoFile) { - const char* argv[] = { - "cdc_rsync.exe", "--ip=1.2.3.4", "--port=1234", "source", - "destination", "--include-from", NULL}; + const char* argv[] = {"cdc_rsync.exe", kSrc, kUserHostDst, "--include-from", + NULL}; EXPECT_FALSE( Parse(static_cast(std::size(argv)) - 1, argv, ¶meters_)); ExpectError(NeedsValueError("include-from")); @@ -441,20 +476,22 @@ TEST_F(ParamsTest, IncludeFrom_NoFile) { TEST_F(ParamsTest, IncludeFrom_ParsesFile) { std::string file = path::Join(base_dir_, "include_files.txt"); - const char* argv[] = { - "cdc_rsync.exe", "--ip=1.2.3.4", "--port=1234", "--include-from", - file.c_str(), "source", "destination", NULL}; + const char* argv[] = {"cdc_rsync.exe", "--include-from", + file.c_str(), kSrc, + kUserHostDst, NULL}; EXPECT_TRUE(Parse(static_cast(std::size(argv)) - 1, argv, ¶meters_)); - ASSERT_EQ(parameters_.filter_rules.size(), 1); - ASSERT_EQ(parameters_.filter_rules[0].type, FilterRule::Type::kInclude); - ASSERT_EQ(parameters_.filter_rules[0].pattern, "file3"); + const std::vector& rules = + parameters_.options.filter.GetRules(); + ASSERT_EQ(rules.size(), 1); + ASSERT_EQ(rules[0].type, PathFilter::Rule::Type::kInclude); + ASSERT_EQ(rules[0].pattern, "file3"); ExpectNoError(); } TEST_F(ParamsTest, ExcludeFrom_NoFile) { - const char* argv[] = {"cdc_rsync.exe", "source", "destination", - "--exclude-from", NULL}; + const char* argv[] = {"cdc_rsync.exe", kSrc, kUserHostDst, "--exclude-from", + NULL}; EXPECT_FALSE( Parse(static_cast(std::size(argv)) - 1, argv, ¶meters_)); ExpectError(NeedsValueError("exclude-from")); @@ -462,16 +499,18 @@ TEST_F(ParamsTest, ExcludeFrom_NoFile) { TEST_F(ParamsTest, ExcludeFrom_ParsesFile) { std::string file = path::Join(base_dir_, "exclude_files.txt"); - const char* argv[] = { - "cdc_rsync.exe", "--ip=1.2.3.4", "--port=1234", "--exclude-from", - file.c_str(), "source", "destination", NULL}; + const char* argv[] = {"cdc_rsync.exe", "--exclude-from", + file.c_str(), kSrc, + kUserHostDst, NULL}; EXPECT_TRUE(Parse(static_cast(std::size(argv)) - 1, argv, ¶meters_)); - ASSERT_EQ(parameters_.filter_rules.size(), 2); - EXPECT_EQ(parameters_.filter_rules[0].type, FilterRule::Type::kExclude); - EXPECT_EQ(parameters_.filter_rules[0].pattern, "file1"); - EXPECT_EQ(parameters_.filter_rules[1].type, FilterRule::Type::kExclude); - EXPECT_EQ(parameters_.filter_rules[1].pattern, "file2"); + const std::vector& rules = + parameters_.options.filter.GetRules(); + ASSERT_EQ(rules.size(), 2); + EXPECT_EQ(rules[0].type, PathFilter::Rule::Type::kExclude); + EXPECT_EQ(rules[0].pattern, "file1"); + EXPECT_EQ(rules[1].type, PathFilter::Rule::Type::kExclude); + EXPECT_EQ(rules[1].pattern, "file2"); ExpectNoError(); } @@ -479,31 +518,31 @@ TEST_F(ParamsTest, IncludeExcludeMixed_ProperOrder) { std::string exclude_file = path::Join(base_dir_, "exclude_files.txt"); std::string include_file = path::Join(base_dir_, "include_files.txt"); const char* argv[] = {"cdc_rsync.exe", - "--ip=1.2.3.4", - "--port=1234", "--include-from", include_file.c_str(), "--exclude=excl1", - "source", + kSrc, "--exclude-from", exclude_file.c_str(), - "destination", + kUserHostDst, "--include", "incl1", NULL}; EXPECT_TRUE(Parse(static_cast(std::size(argv)) - 1, argv, ¶meters_)); - ASSERT_EQ(parameters_.filter_rules.size(), 5); - EXPECT_EQ(parameters_.filter_rules[0].type, FilterRule::Type::kInclude); - EXPECT_EQ(parameters_.filter_rules[0].pattern, "file3"); - EXPECT_EQ(parameters_.filter_rules[1].type, FilterRule::Type::kExclude); - EXPECT_EQ(parameters_.filter_rules[1].pattern, "excl1"); - EXPECT_EQ(parameters_.filter_rules[2].type, FilterRule::Type::kExclude); - EXPECT_EQ(parameters_.filter_rules[2].pattern, "file1"); - EXPECT_EQ(parameters_.filter_rules[3].type, FilterRule::Type::kExclude); - EXPECT_EQ(parameters_.filter_rules[3].pattern, "file2"); - EXPECT_EQ(parameters_.filter_rules[4].type, FilterRule::Type::kInclude); - EXPECT_EQ(parameters_.filter_rules[4].pattern, "incl1"); + const std::vector& rules = + parameters_.options.filter.GetRules(); + ASSERT_EQ(rules.size(), 5); + EXPECT_EQ(rules[0].type, PathFilter::Rule::Type::kInclude); + EXPECT_EQ(rules[0].pattern, "file3"); + EXPECT_EQ(rules[1].type, PathFilter::Rule::Type::kExclude); + EXPECT_EQ(rules[1].pattern, "excl1"); + EXPECT_EQ(rules[2].type, PathFilter::Rule::Type::kExclude); + EXPECT_EQ(rules[2].pattern, "file1"); + EXPECT_EQ(rules[3].type, PathFilter::Rule::Type::kExclude); + EXPECT_EQ(rules[3].pattern, "file2"); + EXPECT_EQ(rules[4].type, PathFilter::Rule::Type::kInclude); + EXPECT_EQ(rules[4].pattern, "incl1"); ExpectNoError(); } diff --git a/cdc_rsync_cli/testdata/params/empty_source_files.txt b/cdc_rsync/testdata/params/empty_source_files.txt similarity index 100% rename from cdc_rsync_cli/testdata/params/empty_source_files.txt rename to cdc_rsync/testdata/params/empty_source_files.txt diff --git a/cdc_rsync_cli/testdata/params/exclude_files.txt b/cdc_rsync/testdata/params/exclude_files.txt similarity index 100% rename from cdc_rsync_cli/testdata/params/exclude_files.txt rename to cdc_rsync/testdata/params/exclude_files.txt diff --git a/cdc_rsync_cli/testdata/params/include_files.txt b/cdc_rsync/testdata/params/include_files.txt similarity index 100% rename from cdc_rsync_cli/testdata/params/include_files.txt rename to cdc_rsync/testdata/params/include_files.txt diff --git a/cdc_rsync_cli/testdata/params/source_files.txt b/cdc_rsync/testdata/params/source_files.txt similarity index 100% rename from cdc_rsync_cli/testdata/params/source_files.txt rename to cdc_rsync/testdata/params/source_files.txt diff --git a/cdc_rsync_cli/.gitignore b/cdc_rsync_cli/.gitignore deleted file mode 100644 index 7dc8dde..0000000 --- a/cdc_rsync_cli/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -x64/* -*.log -*.user \ No newline at end of file diff --git a/cdc_rsync_cli/BUILD b/cdc_rsync_cli/BUILD deleted file mode 100644 index 79ec2af..0000000 --- a/cdc_rsync_cli/BUILD +++ /dev/null @@ -1,44 +0,0 @@ -package(default_visibility = [ - "//:__subpackages__", -]) - -cc_binary( - name = "cdc_rsync", - srcs = ["main.cc"], - deps = [ - ":params", - "//cdc_rsync", - ], -) - -cc_library( - name = "params", - srcs = ["params.cc"], - hdrs = ["params.h"], - deps = [ - "//cdc_rsync", - "@com_github_zstd//:zstd", - "@com_google_absl//absl/status", - ], -) - -cc_test( - name = "params_test", - srcs = ["params_test.cc"], - data = ["testdata/root.txt"] + glob(["testdata/params/**"]), - deps = [ - ":params", - "//common:test_main", - "@com_google_googletest//:gtest", - ], -) - -filegroup( - name = "all_test_sources", - srcs = glob(["*_test.cc"]), -) - -filegroup( - name = "all_test_data", - srcs = glob(["testdata/**"]), -) diff --git a/cdc_rsync_cli/main.cc b/cdc_rsync_cli/main.cc deleted file mode 100644 index a308428..0000000 --- a/cdc_rsync_cli/main.cc +++ /dev/null @@ -1,72 +0,0 @@ -// Copyright 2022 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#define WIN32_LEAN_AND_MEAN -#include - -#include -#include - -#include "cdc_rsync/cdc_rsync.h" -#include "cdc_rsync_cli/params.h" -#include "common/util.h" - -int wmain(int argc, wchar_t* argv[]) { - cdc_ft::params::Parameters parameters; - - // Convert args from wide to UTF8 strings. - std::vector utf8_str_args; - utf8_str_args.reserve(argc); - for (int i = 0; i < argc; i++) { - utf8_str_args.push_back(cdc_ft::Util::WideToUtf8Str(argv[i])); - } - - // Convert args from UTF8 strings to UTF8 c-strings. - std::vector utf8_args; - utf8_args.reserve(argc); - for (const auto& utf8_str_arg : utf8_str_args) { - utf8_args.push_back(utf8_str_arg.c_str()); - } - - if (!cdc_ft::params::Parse(argc, utf8_args.data(), ¶meters)) { - return 1; - } - - // Convert sources from string-vec to c-str-vec. - std::vector sources_ptr; - sources_ptr.reserve(parameters.sources.size()); - for (const std::string& source : parameters.sources) { - sources_ptr.push_back(source.c_str()); - } - - // Convert filter rules from string-structs to c-str-structs. - std::vector filter_rules; - filter_rules.reserve(parameters.filter_rules.size()); - for (const cdc_ft::params::Parameters::FilterRule& rule : - parameters.filter_rules) { - filter_rules.emplace_back(rule.type, rule.pattern.c_str()); - } - - const char* error_message = nullptr; - cdc_ft::ReturnCode code = cdc_ft::Sync( - ¶meters.options, filter_rules.data(), parameters.filter_rules.size(), - parameters.sources_dir.c_str(), sources_ptr.data(), - parameters.sources.size(), parameters.destination.c_str(), - &error_message); - - if (error_message) { - fprintf(stderr, "Error: %s\n", error_message); - } - return static_cast(code); -} diff --git a/cdc_rsync_cli/testdata/root.txt b/cdc_rsync_cli/testdata/root.txt deleted file mode 100644 index e69de29..0000000 diff --git a/cdc_rsync_server/cdc_rsync_server.cc b/cdc_rsync_server/cdc_rsync_server.cc index 07058a6..c167c5e 100644 --- a/cdc_rsync_server/cdc_rsync_server.cc +++ b/cdc_rsync_server/cdc_rsync_server.cc @@ -146,14 +146,14 @@ PathFilter::Rule::Type ToInternalType( } // namespace -GgpRsyncServer::GgpRsyncServer() = default; +CdcRsyncServer::CdcRsyncServer() = default; -GgpRsyncServer::~GgpRsyncServer() { +CdcRsyncServer::~CdcRsyncServer() { message_pump_.reset(); socket_.reset(); } -bool GgpRsyncServer::CheckComponents( +bool CdcRsyncServer::CheckComponents( const std::vector& components) { // Components are expected to reside in the same dir as the executable. std::string component_dir; @@ -172,7 +172,7 @@ bool GgpRsyncServer::CheckComponents( return true; } -absl::Status GgpRsyncServer::Run(int port) { +absl::Status CdcRsyncServer::Run(int port) { socket_ = std::make_unique(); absl::Status status = socket_->StartListening(port); if (!status.ok()) { @@ -205,7 +205,7 @@ absl::Status GgpRsyncServer::Run(int port) { return absl::OkStatus(); } -absl::Status GgpRsyncServer::Sync() { +absl::Status CdcRsyncServer::Sync() { // First, the client sends us options, e.g. the |destination_| directory. absl::Status status = HandleSetOptions(); if (!status.ok()) { @@ -281,7 +281,7 @@ absl::Status GgpRsyncServer::Sync() { return absl::OkStatus(); } -absl::Status GgpRsyncServer::HandleSetOptions() { +absl::Status CdcRsyncServer::HandleSetOptions() { LOG_INFO("Receiving options"); SetOptionsRequest request; @@ -324,7 +324,7 @@ absl::Status GgpRsyncServer::HandleSetOptions() { return absl::OkStatus(); } -absl::Status GgpRsyncServer::FindFiles() { +absl::Status CdcRsyncServer::FindFiles() { Stopwatch stopwatch; FileFinder finder; @@ -350,7 +350,7 @@ absl::Status GgpRsyncServer::FindFiles() { return absl::OkStatus(); } -absl::Status GgpRsyncServer::HandleSendAllFiles() { +absl::Status CdcRsyncServer::HandleSendAllFiles() { std::string current_directory; for (;;) { @@ -385,7 +385,7 @@ absl::Status GgpRsyncServer::HandleSendAllFiles() { } } -absl::Status GgpRsyncServer::DiffFiles() { +absl::Status CdcRsyncServer::DiffFiles() { LOG_INFO("Diffing files"); // Be sure to move the data. It can grow quite large with millions of files. @@ -412,7 +412,7 @@ absl::Status GgpRsyncServer::DiffFiles() { return absl::OkStatus(); } -absl::Status GgpRsyncServer::RemoveExtraneousFilesAndDirs() { +absl::Status CdcRsyncServer::RemoveExtraneousFilesAndDirs() { FileDeleterAndSender deleter(message_pump_.get()); // To guarantee that the folders are empty before they are removed, files are @@ -451,7 +451,7 @@ absl::Status GgpRsyncServer::RemoveExtraneousFilesAndDirs() { return absl::OkStatus(); } -absl::Status GgpRsyncServer::CreateMissingDirs() { +absl::Status CdcRsyncServer::CreateMissingDirs() { for (const DirInfo& dir : diff_.missing_dirs) { // Make directory. std::string path = path::Join(destination_, dir.filepath); @@ -475,7 +475,7 @@ absl::Status GgpRsyncServer::CreateMissingDirs() { } template -absl::Status GgpRsyncServer::SendFileIndices(const char* file_type, +absl::Status CdcRsyncServer::SendFileIndices(const char* file_type, const std::vector& files) { LOG_INFO("Sending indices of missing files to client"); constexpr char error_fmt[] = "Failed to send indices of %s files."; @@ -516,7 +516,7 @@ absl::Status GgpRsyncServer::SendFileIndices(const char* file_type, return absl::OkStatus(); } -absl::Status GgpRsyncServer::HandleSendMissingFileData() { +absl::Status CdcRsyncServer::HandleSendMissingFileData() { if (diff_.missing_files.empty()) { return absl::OkStatus(); } @@ -641,7 +641,7 @@ absl::Status GgpRsyncServer::HandleSendMissingFileData() { return absl::OkStatus(); } -absl::Status GgpRsyncServer::SyncChangedFiles() { +absl::Status CdcRsyncServer::SyncChangedFiles() { if (diff_.changed_files.empty()) { return absl::OkStatus(); } @@ -729,7 +729,7 @@ absl::Status GgpRsyncServer::SyncChangedFiles() { return absl::OkStatus(); } -absl::Status GgpRsyncServer::HandleShutdown() { +absl::Status CdcRsyncServer::HandleShutdown() { ShutdownRequest request; absl::Status status = message_pump_->ReceiveMessage(PacketType::kShutdown, &request); @@ -746,7 +746,7 @@ absl::Status GgpRsyncServer::HandleShutdown() { return absl::OkStatus(); } -void GgpRsyncServer::Thread_OnPackageReceived(PacketType type) { +void CdcRsyncServer::Thread_OnPackageReceived(PacketType type) { if (type != PacketType::kToggleCompression) { return; } diff --git a/cdc_rsync_server/cdc_rsync_server.h b/cdc_rsync_server/cdc_rsync_server.h index 7a75684..0a58549 100644 --- a/cdc_rsync_server/cdc_rsync_server.h +++ b/cdc_rsync_server/cdc_rsync_server.h @@ -33,10 +33,10 @@ namespace cdc_ft { class MessagePump; class ServerSocket; -class GgpRsyncServer { +class CdcRsyncServer { public: - GgpRsyncServer(); - ~GgpRsyncServer(); + CdcRsyncServer(); + ~CdcRsyncServer(); // Checks that the gamelet components (cdc_rsync_server binary etc.) are // up-to-date by checking their sizes and timestamps. diff --git a/cdc_rsync_server/main.cc b/cdc_rsync_server/main.cc index 667ab93..0fef17d 100644 --- a/cdc_rsync_server/main.cc +++ b/cdc_rsync_server/main.cc @@ -48,7 +48,6 @@ ServerExitCode GetExitCode(const absl::Status& status) { case Tag::kSocketEof: // Usually means client disconnected and shut down already. case Tag::kDeployServer: - case Tag::kInstancePickerNotAvailableInQuietMode: case Tag::kConnectionTimeout: case Tag::kCount: // Should not happen in server. @@ -82,7 +81,7 @@ int main(int argc, const char** argv) { cdc_ft::Log::Initialize( std::make_unique(cdc_ft::LogLevel::kWarning)); - cdc_ft::GgpRsyncServer server; + cdc_ft::CdcRsyncServer server; if (!server.CheckComponents(components)) { return cdc_ft::kServerExitCodeOutOfDate; } diff --git a/common/port_manager.h b/common/port_manager.h index 5267e75..34f0a4b 100644 --- a/common/port_manager.h +++ b/common/port_manager.h @@ -51,10 +51,14 @@ class PortManager { // Reserves a port in the range passed to the constructor. The port is // released automatically upon destruction if ReleasePort() is not called // explicitly. - // |timeout_sec| is the timeout for finding available ports on the gamelet - // instance. Returns a DeadlineExceeded error if the timeout is exceeded. + // |check_remote| determines whether the remote port should be checked as + // well. If false, the check is skipped and a port might be returned that is + // still in use remotely. + // |remote_timeout_sec| is the timeout for finding available ports on the + // remote instance. Not used if |check_remote| is false. + // Returns a DeadlineExceeded error if the timeout is exceeded. // Returns a ResourceExhausted error if no ports are available. - absl::StatusOr ReservePort(int timeout_sec); + absl::StatusOr ReservePort(bool check_remote, int remote_timeout_sec); // Releases a reserved port. absl::Status ReleasePort(int port); diff --git a/common/port_manager_test.cc b/common/port_manager_test.cc index ba2d56d..2ec1cd6 100644 --- a/common/port_manager_test.cc +++ b/common/port_manager_test.cc @@ -25,8 +25,8 @@ namespace cdc_ft { namespace { -constexpr int kGameletPort = 12345; -constexpr char kGameletIp[] = "1.2.3.4"; +constexpr int kSshPort = 12345; +constexpr char kUserHost[] = "user@1.2.3.4"; constexpr char kGuid[] = "f77bcdfe-368c-4c45-9f01-230c5e7e2132"; constexpr int kFirstPort = 44450; @@ -38,6 +38,9 @@ constexpr int kTimeoutSec = 1; constexpr char kLocalNetstat[] = "netstat -a -n -p tcp"; constexpr char kRemoteNetstat[] = "netstat --numeric --listening --tcp"; +constexpr bool kCheckRemote = true; +constexpr bool kNoCheckRemote = false; + constexpr char kLocalNetstatOutFmt[] = "TCP 127.0.0.1:50000 127.0.0.1:%i ESTABLISHED"; constexpr char kRemoteNetstatOutFmt[] = @@ -53,7 +56,7 @@ class PortManagerTest : public ::testing::Test { void SetUp() override { Log::Initialize(std::make_unique(LogLevel::kInfo)); - remote_util_.SetIpAndPort(kGameletIp, kGameletPort); + remote_util_.SetUserHostAndPort(kUserHost, kSshPort); } void TearDown() override { Log::Shutdown(); } @@ -70,7 +73,16 @@ TEST_F(PortManagerTest, ReservePortSuccess) { process_factory_.SetProcessOutput(kLocalNetstat, "", "", 0); process_factory_.SetProcessOutput(kRemoteNetstat, "", "", 0); - absl::StatusOr port = port_manager_.ReservePort(kTimeoutSec); + absl::StatusOr port = + port_manager_.ReservePort(kCheckRemote, kTimeoutSec); + ASSERT_OK(port); + EXPECT_EQ(*port, kFirstPort); +} + +TEST_F(PortManagerTest, ReservePortNoRemoteSuccess) { + process_factory_.SetProcessOutput(kLocalNetstat, "", "", 0); + + absl::StatusOr port = port_manager_.ReservePort(kNoCheckRemote, 0); ASSERT_OK(port); EXPECT_EQ(*port, kFirstPort); } @@ -83,7 +95,8 @@ TEST_F(PortManagerTest, ReservePortAllLocalPortsTaken) { process_factory_.SetProcessOutput(kLocalNetstat, local_netstat_out, "", 0); process_factory_.SetProcessOutput(kRemoteNetstat, "", "", 0); - absl::StatusOr port = port_manager_.ReservePort(kTimeoutSec); + absl::StatusOr port = + port_manager_.ReservePort(kCheckRemote, kTimeoutSec); EXPECT_TRUE(absl::IsResourceExhausted(port.status())); EXPECT_TRUE( absl::StrContains(port.status().message(), "No port available in range")); @@ -97,7 +110,8 @@ TEST_F(PortManagerTest, ReservePortAllRemotePortsTaken) { process_factory_.SetProcessOutput(kLocalNetstat, "", "", 0); process_factory_.SetProcessOutput(kRemoteNetstat, remote_netstat_out, "", 0); - absl::StatusOr port = port_manager_.ReservePort(kTimeoutSec); + absl::StatusOr port = + port_manager_.ReservePort(kCheckRemote, kTimeoutSec); EXPECT_TRUE(absl::IsResourceExhausted(port.status())); EXPECT_TRUE( absl::StrContains(port.status().message(), "No port available in range")); @@ -107,7 +121,8 @@ TEST_F(PortManagerTest, ReservePortLocalNetstatFails) { process_factory_.SetProcessOutput(kLocalNetstat, "", "", 1); process_factory_.SetProcessOutput(kRemoteNetstat, "", "", 0); - absl::StatusOr port = port_manager_.ReservePort(kTimeoutSec); + absl::StatusOr port = + port_manager_.ReservePort(kCheckRemote, kTimeoutSec); EXPECT_NOT_OK(port); EXPECT_TRUE( absl::StrContains(port.status().message(), @@ -118,7 +133,8 @@ TEST_F(PortManagerTest, ReservePortRemoteNetstatFails) { process_factory_.SetProcessOutput(kLocalNetstat, "", "", 0); process_factory_.SetProcessOutput(kRemoteNetstat, "", "", 1); - absl::StatusOr port = port_manager_.ReservePort(kTimeoutSec); + absl::StatusOr port = + port_manager_.ReservePort(kCheckRemote, kTimeoutSec); EXPECT_NOT_OK(port); EXPECT_TRUE(absl::StrContains(port.status().message(), "Failed to find available ports on instance")); @@ -129,7 +145,8 @@ TEST_F(PortManagerTest, ReservePortRemoteNetstatTimesOut) { process_factory_.SetProcessNeverExits(kRemoteNetstat); steady_clock_.AutoAdvance(kTimeoutSec * 2 * 1000); - absl::StatusOr port = port_manager_.ReservePort(kTimeoutSec); + absl::StatusOr port = + port_manager_.ReservePort(kCheckRemote, kTimeoutSec); EXPECT_NOT_OK(port); EXPECT_TRUE(absl::IsDeadlineExceeded(port.status())); EXPECT_TRUE(absl::StrContains(port.status().message(), @@ -146,10 +163,14 @@ TEST_F(PortManagerTest, ReservePortMultipleInstances) { // Port managers use shared memory, so different instances know about each // other. This would even work if |port_manager_| and |port_manager2| belonged // to different processes, but we don't test that here. - EXPECT_EQ(*port_manager_.ReservePort(kTimeoutSec), kFirstPort + 0); - EXPECT_EQ(*port_manager2.ReservePort(kTimeoutSec), kFirstPort + 1); - EXPECT_EQ(*port_manager_.ReservePort(kTimeoutSec), kFirstPort + 2); - EXPECT_EQ(*port_manager2.ReservePort(kTimeoutSec), kFirstPort + 3); + EXPECT_EQ(*port_manager_.ReservePort(kCheckRemote, kTimeoutSec), + kFirstPort + 0); + EXPECT_EQ(*port_manager2.ReservePort(kCheckRemote, kTimeoutSec), + kFirstPort + 1); + EXPECT_EQ(*port_manager_.ReservePort(kCheckRemote, kTimeoutSec), + kFirstPort + 2); + EXPECT_EQ(*port_manager2.ReservePort(kCheckRemote, kTimeoutSec), + kFirstPort + 3); } TEST_F(PortManagerTest, ReservePortReusesPortsInLRUOrder) { @@ -157,7 +178,7 @@ TEST_F(PortManagerTest, ReservePortReusesPortsInLRUOrder) { process_factory_.SetProcessOutput(kRemoteNetstat, "", "", 0); for (int n = 0; n < kNumPorts * 2; ++n) { - EXPECT_EQ(*port_manager_.ReservePort(kTimeoutSec), + EXPECT_EQ(*port_manager_.ReservePort(kCheckRemote, kTimeoutSec), kFirstPort + n % kNumPorts); system_clock_.Advance(1000); } @@ -167,10 +188,11 @@ TEST_F(PortManagerTest, ReleasePort) { process_factory_.SetProcessOutput(kLocalNetstat, "", "", 0); process_factory_.SetProcessOutput(kRemoteNetstat, "", "", 0); - absl::StatusOr port = port_manager_.ReservePort(kTimeoutSec); + absl::StatusOr port = + port_manager_.ReservePort(kCheckRemote, kTimeoutSec); EXPECT_EQ(*port, kFirstPort); EXPECT_OK(port_manager_.ReleasePort(*port)); - port = port_manager_.ReservePort(kTimeoutSec); + port = port_manager_.ReservePort(kCheckRemote, kTimeoutSec); EXPECT_EQ(*port, kFirstPort); } @@ -180,10 +202,13 @@ TEST_F(PortManagerTest, ReleasePortOnDestruction) { auto port_manager2 = std::make_unique( kGuid, kFirstPort, kLastPort, &process_factory_, &remote_util_); - EXPECT_EQ(*port_manager2->ReservePort(kTimeoutSec), kFirstPort + 0); - EXPECT_EQ(*port_manager_.ReservePort(kTimeoutSec), kFirstPort + 1); + EXPECT_EQ(*port_manager2->ReservePort(kCheckRemote, kTimeoutSec), + kFirstPort + 0); + EXPECT_EQ(*port_manager_.ReservePort(kCheckRemote, kTimeoutSec), + kFirstPort + 1); port_manager2.reset(); - EXPECT_EQ(*port_manager_.ReservePort(kTimeoutSec), kFirstPort + 0); + EXPECT_EQ(*port_manager_.ReservePort(kCheckRemote, kTimeoutSec), + kFirstPort + 0); } TEST_F(PortManagerTest, FindAvailableLocalPortsSuccess) { diff --git a/common/port_manager_win.cc b/common/port_manager_win.cc index 30093fd..9192456 100644 --- a/common/port_manager_win.cc +++ b/common/port_manager_win.cc @@ -121,7 +121,8 @@ PortManager::~PortManager() { } } -absl::StatusOr PortManager::ReservePort(int timeout_sec) { +absl::StatusOr PortManager::ReservePort(bool check_remote, + int remote_timeout_sec) { // Find available port on workstation. std::unordered_set local_ports; ASSIGN_OR_RETURN(local_ports, @@ -129,13 +130,16 @@ absl::StatusOr PortManager::ReservePort(int timeout_sec) { process_factory_, false), "Failed to find available ports on workstation"); - // Find available port on remote gamelet. - std::unordered_set remote_ports; - ASSIGN_OR_RETURN(remote_ports, - FindAvailableRemotePorts(first_port_, last_port_, "0.0.0.0", - process_factory_, remote_util_, - timeout_sec, false, steady_clock_), - "Failed to find available ports on instance"); + // Find available port on remote instance. + std::unordered_set remote_ports = local_ports; + if (check_remote) { + ASSIGN_OR_RETURN( + remote_ports, + FindAvailableRemotePorts(first_port_, last_port_, "0.0.0.0", + process_factory_, remote_util_, + remote_timeout_sec, false, steady_clock_), + "Failed to find available ports on instance"); + } // Fetch shared memory. void* mem; diff --git a/common/process_win.cc b/common/process_win.cc index b78f8fd..42eb62b 100644 --- a/common/process_win.cc +++ b/common/process_win.cc @@ -58,8 +58,8 @@ absl::Status CreatePipeForOverlappedIo(ScopedHandle* pipe_read_end, ScopedHandle* pipe_write_end) { // We need named pipes for overlapped IO, so create a unique name. int id = g_pipe_serial_number++; - std::string pipe_name = absl::StrFormat( - R"(\\.\Pipe\GgpRsyncIoPipe.%08x.%08x)", GetCurrentProcessId(), id); + std::string pipe_name = absl::StrFormat(R"(\\.\Pipe\CdcIoPipe.%08x.%08x)", + GetCurrentProcessId(), id); // Set the bInheritHandle flag so pipe handles are inherited. SECURITY_ATTRIBUTES security_attributes; diff --git a/common/remote_util.cc b/common/remote_util.cc index 1e85eac..cf5b557 100644 --- a/common/remote_util.cc +++ b/common/remote_util.cc @@ -14,10 +14,9 @@ #include "common/remote_util.h" -#include #include -#include +#include "absl/strings/str_cat.h" #include "absl/strings/str_format.h" #include "common/path.h" #include "common/status.h" @@ -25,32 +24,6 @@ namespace cdc_ft { namespace { -// Escapes command line argument for the Microsoft command line parser in -// preparation for quoting. Double quotes are backslash-escaped. Literal -// backslashes are backslash-escaped if they are followed by a double quote, or -// if they are part of a sequence of backslashes that are followed by a double -// quote. -std::string EscapeForWindows(const std::string& argument) { - std::string str = - std::regex_replace(argument, std::regex(R"(\\*(?=""|$))"), "$1$1"); - return std::regex_replace(str, std::regex("\""), "\\\""); -} - -// Quotes and escapes a command line argument following the convention -// understood by the Microsoft command line parser. -std::string QuoteArgument(const std::string& argument) { - return absl::StrFormat("\"%s\"", EscapeForWindows(argument)); -} - -// Quotes and escapes a command line arguments for use in ssh command. The -// argument is first escaped and quoted for Linux using single quotes and then -// it is escaped to be used by the Microsoft command line parser. -std::string QuoteAndEscapeArgumentForSsh(const std::string& argument) { - std::string quoted_argument = absl::StrFormat( - "'%s'", std::regex_replace(argument, std::regex("'"), "'\\''")); - return EscapeForWindows(quoted_argument); -} - // Gets the argument for SSH (reverse) port forwarding, e.g. -L23:localhost:45. std::string GetPortForwardingArg(int local_port, int remote_port, bool reverse) { @@ -69,21 +42,29 @@ RemoteUtil::RemoteUtil(int verbosity, bool quiet, process_factory_(process_factory), forward_output_to_log_(forward_output_to_log) {} -void RemoteUtil::SetIpAndPort(const std::string& gamelet_ip, int ssh_port) { - gamelet_ip_ = gamelet_ip; - ssh_port_ = ssh_port; +void RemoteUtil::SetUserHostAndPort(std::string user_host, int port) { + user_host_ = std::move(user_host); + ssh_port_ = port; +} +void RemoteUtil::SetScpCommand(std::string scp_command) { + scp_command_ = std::move(scp_command); +} + +void RemoteUtil::SetSshCommand(std::string ssh_command) { + ssh_command_ = std::move(ssh_command); } absl::Status RemoteUtil::Scp(std::vector source_filepaths, const std::string& dest, bool compress) { - absl::Status status = CheckIpPort(); + absl::Status status = CheckHostPort(); if (!status.ok()) { return status; } std::string source_args; for (const std::string& sourceFilePath : source_filepaths) { - source_args += QuoteArgument(sourceFilePath) + " "; + // Workaround for scp thinking that C is a host in C:\path\to\foo. + source_args += QuoteArgument("//./" + sourceFilePath) + " "; } // -p preserves timestamps. This enables timestamp-based up-to-date checks. @@ -91,18 +72,10 @@ absl::Status RemoteUtil::Scp(std::vector source_filepaths, start_info.command = absl::StrFormat( "%s " "%s %s -p -T " - "-F %s " - "-i %s -P %i " - "-oStrictHostKeyChecking=yes " - "-oUserKnownHostsFile=\"\"\"%s\"\"\" %s " - "cloudcast@%s:" + "-P %i %s " "%s", - QuoteArgument(sdk_util_.GetScpExePath()), - quiet_ || verbosity_ < 2 ? "-q" : "", compress ? "-C" : "", - QuoteArgument(sdk_util_.GetSshConfigPath()), - QuoteArgument(sdk_util_.GetSshKeyFilePath()), ssh_port_, - sdk_util_.GetSshKnownHostsFilePath(), source_args, - QuoteArgument(gamelet_ip_), QuoteAndEscapeArgumentForSsh(dest)); + scp_command_, quiet_ || verbosity_ < 2 ? "-q" : "", compress ? "-C" : "", + ssh_port_, source_args, QuoteArgument(user_host_ + ":" + dest)); start_info.name = "scp"; start_info.forward_output_to_log = forward_output_to_log_; @@ -111,7 +84,7 @@ absl::Status RemoteUtil::Scp(std::vector source_filepaths, absl::Status RemoteUtil::Sync(std::vector source_filepaths, const std::string& dest) { - absl::Status status = CheckIpPort(); + absl::Status status = CheckHostPort(); if (!status.ok()) { return status; } @@ -123,9 +96,9 @@ absl::Status RemoteUtil::Sync(std::vector source_filepaths, ProcessStartInfo start_info; start_info.command = absl::StrFormat( - "%s --ip=%s --port=%i -z %s %s%s", - path::Join(sdk_util_.GetDevBinPath(), "cdc_rsync"), - QuoteArgument(gamelet_ip_), ssh_port_, + "cdc_rsync --ip=%s --port=%i -z " + "%s %s%s", + QuoteArgument(user_host_), ssh_port_, quiet_ || verbosity_ < 2 ? "-q " : " ", source_args, QuoteArgument(dest)); start_info.name = "cdc_rsync"; start_info.forward_output_to_log = forward_output_to_log_; @@ -135,16 +108,16 @@ absl::Status RemoteUtil::Sync(std::vector source_filepaths, absl::Status RemoteUtil::Chmod(const std::string& mode, const std::string& remote_path, bool quiet) { - std::string remote_command = absl::StrFormat( - "chmod %s %s %s", QuoteArgument(mode), - QuoteAndEscapeArgumentForSsh(remote_path), quiet ? "-f" : ""); + std::string remote_command = + absl::StrFormat("chmod %s %s %s", QuoteArgument(mode), + EscapeForWindows(remote_path), quiet ? "-f" : ""); return Run(remote_command, "chmod"); } absl::Status RemoteUtil::Rm(const std::string& remote_path, bool force) { - std::string remote_command = absl::StrFormat( - "rm %s %s", force ? "-f" : "", QuoteAndEscapeArgumentForSsh(remote_path)); + std::string remote_command = absl::StrFormat("rm %s %s", force ? "-f" : "", + EscapeForWindows(remote_path)); return Run(remote_command, "rm"); } @@ -152,14 +125,14 @@ absl::Status RemoteUtil::Rm(const std::string& remote_path, bool force) { absl::Status RemoteUtil::Mv(const std::string& old_remote_path, const std::string& new_remote_path) { std::string remote_command = - absl::StrFormat("mv %s %s", QuoteAndEscapeArgumentForSsh(old_remote_path), - QuoteAndEscapeArgumentForSsh(new_remote_path)); + absl::StrFormat("mv %s %s", EscapeForWindows(old_remote_path), + EscapeForWindows(new_remote_path)); return Run(remote_command, "mv"); } absl::Status RemoteUtil::Run(std::string remote_command, std::string name) { - absl::Status status = CheckIpPort(); + absl::Status status = CheckHostPort(); if (!status.ok()) { return status; } @@ -201,25 +174,37 @@ ProcessStartInfo RemoteUtil::BuildProcessStartInfoForSshInternal( start_info.command = absl::StrFormat( "%s " "%s -tt " - "-F %s " - "-i %s " "-oServerAliveCountMax=6 " // Number of lost msgs before ssh terminates "-oServerAliveInterval=5 " // Time interval between alive msgs - "-oStrictHostKeyChecking=yes " - "-oUserKnownHostsFile=\"\"\"%s\"\"\" %s" - "cloudcast@%s -p %i %s", - QuoteArgument(sdk_util_.GetSshExePath()), - quiet_ || verbosity_ < 2 ? "-q" : "", - QuoteArgument(sdk_util_.GetSshConfigPath()), - QuoteArgument(sdk_util_.GetSshKeyFilePath()), - sdk_util_.GetSshKnownHostsFilePath(), forward_arg, - QuoteArgument(gamelet_ip_), ssh_port_, remote_command_arg); + "%s %s -p %i %s", + ssh_command_, quiet_ || verbosity_ < 2 ? "-q" : "", forward_arg, + QuoteArgument(user_host_), ssh_port_, remote_command_arg); start_info.forward_output_to_log = forward_output_to_log_; return start_info; } -absl::Status RemoteUtil::CheckIpPort() { - if (gamelet_ip_.empty() || ssh_port_ == 0) { +std::string RemoteUtil::EscapeForWindows(const std::string& argument) { + std::string str = + std::regex_replace(argument, std::regex(R"(\\*(?="|$))"), "$&$&"); + return std::regex_replace(str, std::regex(R"(")"), R"(\")"); +} + +std::string RemoteUtil::QuoteArgument(const std::string& argument) { + return absl::StrCat("\"", EscapeForWindows(argument), "\""); +} + +std::string RemoteUtil::QuoteArgumentForSsh(const std::string& argument) { + return absl::StrFormat( + "'%s'", std::regex_replace(argument, std::regex("'"), "'\\''")); +} + +std::string RemoteUtil::QuoteAndEscapeArgumentForSsh( + const std::string& argument) { + return EscapeForWindows(QuoteArgumentForSsh(argument)); +} + +absl::Status RemoteUtil::CheckHostPort() { + if (user_host_.empty() || ssh_port_ == 0) { return MakeStatus("IP or port not set"); } diff --git a/common/remote_util.h b/common/remote_util.h index 936ebdb..31e59b3 100644 --- a/common/remote_util.h +++ b/common/remote_util.h @@ -22,7 +22,6 @@ #include "absl/status/status.h" #include "common/process.h" -#include "common/sdk_util.h" namespace cdc_ft { @@ -30,6 +29,8 @@ namespace cdc_ft { // Windows-only. class RemoteUtil { public: + static constexpr int kDefaultSshPort = 22; + // If |verbosity| is > 0 and |quiet| is false, output from scp, ssh etc. // commands is shown. // If |quiet| is true, scp, ssh etc. commands use quiet mode. @@ -38,61 +39,67 @@ class RemoteUtil { RemoteUtil(int verbosity, bool quiet, ProcessFactory* process_factory, bool forward_output_to_log); - // Returns the initialization status. Should be OK unless in case of some rare - // internal error. Should be checked before accessing any members. - const absl::Status& GetInitStatus() const { - return sdk_util_.GetInitStatus(); - } + // Sets the SSH username and hostname of the remote instance, as well as the + // SSH tunnel port. |user_host| must be of the form [user@]host. + void SetUserHostAndPort(std::string user_host, int port); - // Set IP of the remote instance and the ssh tunnel port. - void SetIpAndPort(const std::string& gamelet_ip, int ssh_port); + // Sets the SCP command binary path and additional arguments, e.g. + // C:\path\to\scp.exe -F -i + // -oStrictHostKeyChecking=yes -oUserKnownHostsFile="""file""" + // By default, searches scp.exe on the path environment variables. + void SetScpCommand(std::string scp_command); + + // Sets the SSH command binary path and additional arguments, e.g. + // C:\path\to\ssh.exe -F -i + // -oStrictHostKeyChecking=yes -oUserKnownHostsFile="""file""" + // By default, searches ssh.exe on the path environment variables. + void SetSshCommand(std::string ssh_command); // Copies |source_filepaths| to the remote folder |dest| on the gamelet using - // scp. Must call either InitSsh or SetGameletIp before calling this method. - // If |compress| is true, compressed upload is used. + // scp. If |compress| is true, compressed upload is used. + // Must call SetUserHostAndPort before calling this method. absl::Status Scp(std::vector source_filepaths, const std::string& dest, bool compress); // Syncs |source_filepaths| to the remote folder |dest| on the gamelet using - // cdc_rsync. Must call either InitSsh or SetGameletIp before calling this - // method. + // cdc_rsync. Must call SetUserHostAndPort before calling this method. absl::Status Sync(std::vector source_filepaths, const std::string& dest); // Calls 'chmod |mode| |remote_path|' on the gamelet. - // Must call either InitSsh or SetGameletIp before calling this method. + // Must call SetUserHostAndPort before calling this method. absl::Status Chmod(const std::string& mode, const std::string& remote_path, bool quiet = false); // Calls 'rm [-f] |remote_path|' on the gamelet. - // Must call either InitSsh or SetGameletIp before calling this method. + // Must call SetUserHostAndPort before calling this method. absl::Status Rm(const std::string& remote_path, bool force); // Calls `mv |old_remote_path| |new_remote_path| on the gamelet. - // Must call either InitSsh or SetGameletIp before calling this method. + // Must call SetUserHostAndPort before calling this method. absl::Status Mv(const std::string& old_remote_path, const std::string& new_remote_path); // Runs |remote_command| on the gamelet. The command must be properly escaped. // |name| is the name of the command displayed in the logs. - // Must call either InitSsh or SetGameletIp before calling this method. + // Must call SetUserHostAndPort before calling this method. absl::Status Run(std::string remote_command, std::string name); - // Builds an ssh command that executes |remote_command| on the gamelet. + // Builds an SSH command that executes |remote_command| on the gamelet. ProcessStartInfo BuildProcessStartInfoForSsh(std::string remote_command); - // Builds an ssh command that runs SSH port forwarding to the gamelet, using + // Builds an SSH command that runs SSH port forwarding to the gamelet, using // the given |local_port| and |remote_port|. // If |reverse| is true, sets up reverse port forwarding. - // Must call either InitSsh or SetGameletIp before calling this method. + // Must call SetUserHostAndPort before calling this method. ProcessStartInfo BuildProcessStartInfoForSshPortForward(int local_port, int remote_port, bool reverse); - // Builds an ssh command that executes |remote_command| on the gamelet, using + // Builds an SSH command that executes |remote_command| on the gamelet, using // port forwarding with given |local_port| and |remote_port|. // If |reverse| is true, sets up reverse port forwarding. - // Must call either InitSsh or SetGameletIp before calling this method. + // Must call SetUserHostAndPort before calling this method. ProcessStartInfo BuildProcessStartInfoForSshPortForwardAndCommand( int local_port, int remote_port, bool reverse, std::string remote_command); @@ -100,9 +107,28 @@ class RemoteUtil { // Returns whether output is suppressed. bool Quiet() const { return quiet_; } + // Escapes command line argument for the Microsoft command line parser in + // preparation for quoting. Double quotes are backslash-escaped. One or more + // backslashes are backslash-escaped if they are followed by a double quote, + // or if they occur at the end of the string, e.g. + // foo\bar -> foo\bar, foo\ -> foo\\, foo\\"bar -> foo\\\\\"bar. + static std::string EscapeForWindows(const std::string& argument); + + // Quotes and escapes a command line argument following the convention + // understood by the Microsoft command line parser. + static std::string QuoteArgument(const std::string& argument); + + // Quotes and escapes a command line argument for usage in SSH. + static std::string QuoteArgumentForSsh(const std::string& argument); + + // Quotes and escapes a command line arguments for use in SSH command. The + // argument is first escaped and quoted for Linux using single quotes and then + // it is escaped to be used by the Microsoft command line parser. + static std::string QuoteAndEscapeArgumentForSsh(const std::string& argument); + private: - // Verifies that both |gamelet_ip_| and |ssh_port_| are set. - absl::Status CheckIpPort(); + // Verifies that both || and |ssh_port_| are set. + absl::Status CheckHostPort(); // Common code for BuildProcessStartInfoForSsh*. ProcessStartInfo BuildProcessStartInfoForSshInternal( @@ -113,9 +139,10 @@ class RemoteUtil { ProcessFactory* const process_factory_; const bool forward_output_to_log_; - SdkUtil sdk_util_; - std::string gamelet_ip_; - int ssh_port_ = 0; + std::string scp_command_ = "scp"; + std::string ssh_command_ = "ssh"; + std::string user_host_; + int ssh_port_ = kDefaultSshPort; }; } // namespace cdc_ft diff --git a/common/remote_util_test.cc b/common/remote_util_test.cc index 3befbce..7ac730a 100644 --- a/common/remote_util_test.cc +++ b/common/remote_util_test.cc @@ -21,11 +21,11 @@ namespace cdc_ft { namespace { -constexpr int kGameletPort = 12345; -constexpr char kGameletPortArg[] = "-p 12345"; +constexpr int kSshPort = 12345; +constexpr char kSshPortArg[] = "-p 12345"; -constexpr char kGameletIp[] = "1.2.3.4"; -constexpr char kGameletIpArg[] = "cloudcast@\"1.2.3.4\""; +constexpr char kUserHost[] = "user@example.com"; +constexpr char kUserHostArg[] = "\"user@example.com\""; constexpr int kLocalPort = 23456; constexpr int kRemotePort = 34567; @@ -44,7 +44,7 @@ class RemoteUtilTest : public ::testing::Test { void SetUp() override { Log::Initialize(std::make_unique(LogLevel::kInfo)); - util_.SetIpAndPort(kGameletIp, kGameletPort); + util_.SetUserHostAndPort(kUserHost, kSshPort); } void TearDown() override { Log::Shutdown(); } @@ -64,42 +64,68 @@ class RemoteUtilTest : public ::testing::Test { TEST_F(RemoteUtilTest, BuildProcessStartInfoForSsh) { ProcessStartInfo si = util_.BuildProcessStartInfoForSsh(kCommand); - ExpectContains(si.command, - {"ssh.exe", "GGP\\ssh\\id", "oStrictHostKeyChecking=yes", - "oUserKnownHostsFile", "known_hosts", kGameletPortArg, - kGameletIpArg, kCommand}); + ExpectContains(si.command, {"ssh", kSshPortArg, kUserHostArg, kCommand}); } TEST_F(RemoteUtilTest, BuildProcessStartInfoForSshPortForward) { ProcessStartInfo si = util_.BuildProcessStartInfoForSshPortForward( kLocalPort, kRemotePort, kRegular); ExpectContains(si.command, - {"ssh.exe", "GGP\\ssh\\id", "oStrictHostKeyChecking=yes", - "oUserKnownHostsFile", "known_hosts", kGameletPortArg, - kGameletIpArg, kPortForwardingArg}); + {"ssh", kSshPortArg, kUserHostArg, kPortForwardingArg}); si = util_.BuildProcessStartInfoForSshPortForward(kLocalPort, kRemotePort, kReverse); ExpectContains(si.command, - {"ssh.exe", "GGP\\ssh\\id", "oStrictHostKeyChecking=yes", - "oUserKnownHostsFile", "known_hosts", kGameletPortArg, - kGameletIpArg, kReversePortForwardingArg}); + {"ssh", kSshPortArg, kUserHostArg, kReversePortForwardingArg}); } TEST_F(RemoteUtilTest, BuildProcessStartInfoForSshPortForwardAndCommand) { ProcessStartInfo si = util_.BuildProcessStartInfoForSshPortForwardAndCommand( kLocalPort, kRemotePort, kRegular, kCommand); - ExpectContains(si.command, - {"ssh.exe", "GGP\\ssh\\id", "oStrictHostKeyChecking=yes", - "oUserKnownHostsFile", "known_hosts", kGameletPortArg, - kGameletIpArg, kPortForwardingArg, kCommand}); + ExpectContains(si.command, {"ssh", kSshPortArg, kUserHostArg, + kPortForwardingArg, kCommand}); si = util_.BuildProcessStartInfoForSshPortForwardAndCommand( kLocalPort, kRemotePort, kReverse, kCommand); - ExpectContains(si.command, - {"ssh.exe", "GGP\\ssh\\id", "oStrictHostKeyChecking=yes", - "oUserKnownHostsFile", "known_hosts", kGameletPortArg, - kGameletIpArg, kReversePortForwardingArg, kCommand}); + ExpectContains(si.command, {"ssh", kSshPortArg, kUserHostArg, + kReversePortForwardingArg, kCommand}); +} +TEST_F(RemoteUtilTest, BuildProcessStartInfoForSshWithCustomCommand) { + constexpr char kCustomSshCmd[] = "C:\\path\\to\\ssh.exe --fooarg --bararg=42"; + util_.SetSshCommand(kCustomSshCmd); + ProcessStartInfo si = util_.BuildProcessStartInfoForSsh(kCommand); + ExpectContains(si.command, {kCustomSshCmd}); +} + +TEST_F(RemoteUtilTest, EscapeForWindows) { + EXPECT_EQ("foo", RemoteUtil::EscapeForWindows("foo")); + EXPECT_EQ("foo bar", RemoteUtil::EscapeForWindows("foo bar")); + EXPECT_EQ("foo\\bar", RemoteUtil::EscapeForWindows("foo\\bar")); + EXPECT_EQ("\\\\foo", RemoteUtil::EscapeForWindows("\\\\foo")); + EXPECT_EQ("foo\\\\", RemoteUtil::EscapeForWindows("foo\\")); + EXPECT_EQ("foo\\\\\\\\", RemoteUtil::EscapeForWindows("foo\\\\")); + EXPECT_EQ("foo\\\"", RemoteUtil::EscapeForWindows("foo\"")); + EXPECT_EQ("foo\\\"bar", RemoteUtil::EscapeForWindows("foo\"bar")); + EXPECT_EQ("foo\\\\\\\"bar", RemoteUtil::EscapeForWindows("foo\\\"bar")); + EXPECT_EQ("foo\\\\\\\\\\\"bar", RemoteUtil::EscapeForWindows("foo\\\\\"bar")); + EXPECT_EQ("\\\"foo\\\"", RemoteUtil::EscapeForWindows("\"foo\"")); + EXPECT_EQ("\\\" \\file.txt", RemoteUtil::EscapeForWindows("\" \\file.txt")); +} + +TEST_F(RemoteUtilTest, QuoteArgument) { + EXPECT_EQ("\"foo\"", RemoteUtil::QuoteArgument("foo")); + EXPECT_EQ("\"foo bar\"", RemoteUtil::QuoteArgument("foo bar")); + EXPECT_EQ("\"foo\\bar\"", RemoteUtil::QuoteArgument("foo\\bar")); + EXPECT_EQ("\"\\\\foo\"", RemoteUtil::QuoteArgument("\\\\foo")); + EXPECT_EQ("\"foo\\\\\"", RemoteUtil::QuoteArgument("foo\\")); + EXPECT_EQ("\"foo\\\\\\\\\"", RemoteUtil::QuoteArgument("foo\\\\")); + EXPECT_EQ("\"foo\\\"\"", RemoteUtil::QuoteArgument("foo\"")); + EXPECT_EQ("\"foo\\\"bar\"", RemoteUtil::QuoteArgument("foo\"bar")); + EXPECT_EQ("\"foo\\\\\\\"bar\"", RemoteUtil::QuoteArgument("foo\\\"bar")); + EXPECT_EQ("\"foo\\\\\\\\\\\"bar\"", + RemoteUtil::QuoteArgument("foo\\\\\"bar")); + EXPECT_EQ("\"\\\"foo\\\"\"", RemoteUtil::QuoteArgument("\"foo\"")); + EXPECT_EQ("\"\\\" \\file.txt\"", RemoteUtil::QuoteArgument("\" \\file.txt")); } } // namespace diff --git a/common/sdk_util.cc b/common/sdk_util.cc index fc9cb1c..453c497 100644 --- a/common/sdk_util.cc +++ b/common/sdk_util.cc @@ -31,13 +31,6 @@ SdkUtil::SdkUtil() { absl::Status status = path::GetEnv("GGP_SDK_PATH", &ggp_sdk_path_env_); if (absl::IsNotFound(status) || ggp_sdk_path_env_.empty()) ggp_sdk_path_env_ = path::Join(program_files_path_, "GGP SDK"); - - // Create an empty config file if it does not exist yet. - const std::string ssh_config_path = GetSshConfigPath(); - if (!path::Exists(ssh_config_path)) { - init_status_.Update(path::CreateDirRec(path::DirName(ssh_config_path))); - init_status_.Update(path::WriteFile(ssh_config_path, nullptr, 0)); - } } SdkUtil::~SdkUtil() = default; @@ -57,37 +50,8 @@ std::string SdkUtil::GetLogPath(const char* log_base_name) const { return path::Join(GetUserConfigPath(), "logs", log_base_name + timestamp_ext); } -std::string SdkUtil::GetSshConfigPath() const { - return path::Join(GetUserConfigPath(), "ssh", "config"); -} - -std::string SdkUtil::GetSshKeyFilePath() const { - return path::Join(GetUserConfigPath(), "ssh", "id_rsa"); -} - -std::string SdkUtil::GetSshKnownHostsFilePath() const { - return path::Join(GetUserConfigPath(), "ssh", "known_hosts"); -} - -std::string SdkUtil::GetSDKPath() const { - assert(init_status_.ok()); - return ggp_sdk_path_env_; -} - std::string SdkUtil::GetDevBinPath() const { - return path::Join(GetSDKPath(), "dev", "bin"); -} - -std::string SdkUtil::GetSshPath() const { - return path::Join(GetSDKPath(), "tools", "OpenSSH-Win64"); -} - -std::string SdkUtil::GetSshExePath() const { - return path::Join(GetSshPath(), "ssh.exe"); -} - -std::string SdkUtil::GetScpExePath() const { - return path::Join(GetSshPath(), "scp.exe"); + return path::Join(ggp_sdk_path_env_, "dev", "bin"); } } // namespace cdc_ft diff --git a/common/sdk_util.h b/common/sdk_util.h index b42c036..5b2cbe3 100644 --- a/common/sdk_util.h +++ b/common/sdk_util.h @@ -51,38 +51,10 @@ class SdkUtil { // %APPDATA%\GGP\logs\log_base_name.20210729-125930.log. std::string GetLogPath(const char* log_base_name) const; - // Returns the path of the ssh configuration file, e.g. - // %APPDATA%\GGP\ssh\config. - std::string GetSshConfigPath() const; - - // Returns the path of the ssh private key file in the SDK configuration, e.g. - // %APPDATA%\GGP\ssh\id_rsa. - std::string GetSshKeyFilePath() const; - - // Returns the path of the ssh known_hosts file in the SDK configuration, e.g. - // %APPDATA%\GGP\ssh\known_hosts. - std::string GetSshKnownHostsFilePath() const; - - // Returns the path of the installed SDK, e.g. - // C:\Program Files\GGP SDK. - std::string GetSDKPath() const; - // Returns the path of the dev tools that ship with the SDK, e.g. // C:\Program Files\GGP SDK\dev\bin. std::string GetDevBinPath() const; - // Returns the path of the OpenSSH tools that ship with the SDK, e.g. - // C:\Program Files\GGP SDK\tools\OpenSSH-Win64. - std::string GetSshPath() const; - - // Returns the path of ssh.exe that ships with the SDK, e.g. - // C:\Program Files\GGP SDK\tools\OpenSSH-Win64\ssh.exe. - std::string GetSshExePath() const; - - // Returns the path of scp.exe that ships with the SDK, e.g. - // C:\Program Files\GGP SDK\tools\OpenSSH-Win64\scp.exe. - std::string GetScpExePath() const; - private: std::string roaming_appdata_path_; std::string program_files_path_; diff --git a/common/sdk_util_test.cc b/common/sdk_util_test.cc index 911834c..53574ee 100644 --- a/common/sdk_util_test.cc +++ b/common/sdk_util_test.cc @@ -42,13 +42,6 @@ class SdkUtilTest : public ::testing::Test { protected: void CheckSdkPaths(const SdkUtil& sdk_util, const std::string& sdk_dir) { - EXPECT_EQ(sdk_util.GetSDKPath(), sdk_dir); - EXPECT_EQ(sdk_util.GetSshPath(), - path::Join(sdk_dir, "tools\\OpenSSH-Win64")); - EXPECT_EQ(sdk_util.GetSshExePath(), - path::Join(sdk_dir, "tools\\OpenSSH-Win64\\ssh.exe")); - EXPECT_EQ(sdk_util.GetScpExePath(), - path::Join(sdk_dir, "tools\\OpenSSH-Win64\\scp.exe")); EXPECT_EQ(sdk_util.GetDevBinPath(), path::Join(sdk_dir, "dev", "bin")); } @@ -81,11 +74,6 @@ TEST_F(SdkUtilTest, CheckRoamingAppDataPaths) { const std::string ggp_path = path::Join(appdata_dir, "GGP"); EXPECT_EQ(sdk_util.GetUserConfigPath(), ggp_path); EXPECT_EQ(sdk_util.GetServicesConfigPath(), path::Join(ggp_path, "services")); - EXPECT_EQ(sdk_util.GetSshConfigPath(), path::Join(ggp_path, "ssh", "config")); - EXPECT_EQ(sdk_util.GetSshKeyFilePath(), - path::Join(ggp_path, "ssh", "id_rsa")); - EXPECT_EQ(sdk_util.GetSshKnownHostsFilePath(), - path::Join(ggp_path, "ssh", "known_hosts")); } TEST_F(SdkUtilTest, CheckSdkPathsWithoutGgpSdkPathEnv) { diff --git a/common/status.h b/common/status.h index 6dce7ff..d2febfa 100644 --- a/common/status.h +++ b/common/status.h @@ -77,14 +77,11 @@ enum class Tag : uint8_t { // The gamelet components need to be re-deployed. kDeployServer = 2, - // Something asks for user input, but we're in quiet mode. - kInstancePickerNotAvailableInQuietMode = 3, - // Timeout while trying to connect to the gamelet component. - kConnectionTimeout = 4, + kConnectionTimeout = 3, // MUST BE LAST. - kCount = 5, + kCount = 4, }; // Tags a status. No-op if |status| is OK. Overwrites existing tags. diff --git a/file_transfer.sln b/file_transfer.sln index 573245c..755fce1 100644 --- a/file_transfer.sln +++ b/file_transfer.sln @@ -4,7 +4,7 @@ VisualStudioVersion = 16.0.31702.278 MinimumVisualStudioVersion = 10.0.40219.1 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "CDC RSync", "CDC RSync", "{74FA49B8-56C3-4F9E-B9D5-35FA1C9A13C8}" EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "cdc_rsync_cli", "cdc_rsync_cli\cdc_rsync_cli.vcxproj", "{3FAC852A-00A8-4CFB-9160-07EFF2B73562}" +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "cdc_rsync", "cdc_rsync\cdc_rsync.vcxproj", "{3FAC852A-00A8-4CFB-9160-07EFF2B73562}" EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "cdc_rsync_server", "cdc_rsync_server\cdc_rsync_server.vcxproj", "{4ECE65E0-D950-4B96-8AD5-0313261B8C8D}" EndProject diff --git a/tests_cdc_rsync/BUILD b/tests_cdc_rsync/BUILD index 833ca53..010c668 100644 --- a/tests_cdc_rsync/BUILD +++ b/tests_cdc_rsync/BUILD @@ -15,24 +15,22 @@ cc_binary( srcs = [ "//cdc_rsync:all_test_sources", "//cdc_rsync/base:all_test_sources", - "//cdc_rsync_cli:all_test_sources", "//cdc_rsync_server:all_test_sources", ], data = [ "//cdc_rsync:all_test_data", "//cdc_rsync/base:all_test_data", - "//cdc_rsync_cli:all_test_data", "//cdc_rsync_server:all_test_data", ], deps = [ "//cdc_rsync:file_finder_and_sender", "//cdc_rsync:parallel_file_opener", + "//cdc_rsync:params", "//cdc_rsync:progress_tracker", "//cdc_rsync:zstd_stream", "//cdc_rsync/base:cdc_interface", "//cdc_rsync/base:fake_socket", "//cdc_rsync/base:message_pump", - "//cdc_rsync_cli:params", "//cdc_rsync_server:file_deleter_and_sender", "//cdc_rsync_server:file_diff_generator", "//cdc_rsync_server:file_finder", diff --git a/tools/windows_cc_library.bzl b/tools/windows_cc_library.bzl deleted file mode 100644 index 0129b00..0000000 --- a/tools/windows_cc_library.bzl +++ /dev/null @@ -1,91 +0,0 @@ -# Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -""" -This is a simple cc_windows_shared_library rule for builing a DLL on Windows -that other cc rules can depend on. - -Example useage: - cc_windows_shared_library( - name = "hellolib", - srcs = [ - "hello-library.cpp", - ], - hdrs = ["hello-library.h"], - # Use this to distinguish compiling vs. linking against the DLL. - copts = ["/DCOMPILING_DLL"], - ) - -Define COMPILING_DLL to export symbols in the header when compiling the DLL as -follows: - - #ifdef COMPILING_DLL - #define DLLEXPORT __declspec(dllexport) - #else - #define DLLEXPORT __declspec(dllimport) - #endif - - DLLEXPORT void foo(); - -For more information and sample usage, see: -https://github.com/bazelbuild/bazel/blob/master/examples/windows/dll/ -""" - -load("@rules_cc//cc:defs.bzl", "cc_binary", "cc_import", "cc_library") - -def cc_windows_shared_library( - name, - srcs = [], - deps = [], - hdrs = [], - visibility = None, - **kwargs): - """A simple cc_windows_shared_library rule for builing a Windows DLL.""" - dll_name = name + ".dll" - import_lib_name = name + "_import_lib" - import_target_name = name + "_dll_import" - - # Building a shared library requires a cc_binary with linkshared = 1 set. - cc_binary( - name = dll_name, - srcs = srcs + hdrs, - deps = deps, - linkshared = 1, - **kwargs - ) - - # Get the import library for the dll - native.filegroup( - name = import_lib_name, - srcs = [":" + dll_name], - output_group = "interface_library", - ) - - # Because we cannot directly depend on cc_binary from other cc rules in deps attribute, - # we use cc_import as a bridge to depend on the dll. - cc_import( - name = import_target_name, - interface_library = ":" + import_lib_name, - shared_library = ":" + dll_name, - ) - - # Create a new cc_library to also include the headers needed for the shared library - cc_library( - name = name, - hdrs = hdrs, - visibility = visibility, - deps = deps + [ - ":" + import_target_name, - ], - )