github-mirrors/netris-cdc-file-transfer
1
0
Fork 0
mirror of https://github.com/nestriness/cdc-file-transfer.git synced 2026-05-01 17:03:07 +03:00
Code Issues Packages Projects Releases Wiki Activity

Remove GGP dependencies from CDC RSync (#1)

Browse Source 
* Remove dependencies of cdc_sync from GGP

Allows overriding the SSH and SCP commands via command line flags.
Hence, strict host checking, SSH config etc. can be removed since it
is passed in by command line flags for GGP. Also deploys
cdc_rsync_server to ~/.cache/cdc_file_transfer/ and creates that dir
if it does not exist.

* Tweak RemoteUtil

Replaces localhost: by //./ in the workaround for scp since localhost:
had two disadvantages: 1) It required 2 gnubby touches for gLinux and
2) it didn't work for ggp. //./ works for both. Also tweaks quoting,
which didn't quite work for ggp.

* Don't check remote ports in cdc_rsync

Turns off checking remote ports in PortManager. In the future, the
server should return available ports after failing to connect to the
provided port.

Since now the first remote connection is running cdc_rsync_server,
the timeout check has to be done when running that process.

* Remove now-unused kInstancePickerNotAvailableInQuietMode enum

* Add more details to the readme

* [cdc_rsync] Accept [user@]host:destination

Removes the --ip command line argument and assumes user/host are
passed in along with the destination, so it works in the same way as
other popular tools.

* [ggp_rsync] Combine server deploy commands

Combines two chmod and one mv command into one ssh command. This makes
deploy a bit quicker, especially if each ssh command involves touching
your gnubby.

* Remove GGP specific stuff from VS build commands

* [cdc_rsync] Get rid of cdc_rsync.dll

Compile the CDC RSync client as a static library instead. This removes
quite a bit of boiler plate and makes string handling easier since
we can now pass std::strings instead of const chars.

Also fixes an issue where we were sometimes trying to assign nullptr
to std::strings, which is forbidden.

* Allow specifying ssh/scp commands with env vars

* Rename GgpRsync* to CdcRsync*

* Merge ggp_rsync_cli into ggp_rsync

* [cdc_rsync] Refactor cdc_rsync.cc/h

Merges cdc_rsync.cc/h with main.cc and CdcRsyncClient since code is
closer to where it's being used and should be more readable.
This commit is contained in:
ljusten
2022-11-15 12:48:09 +01:00
committed by GitHub
parent 4326e972ac
commit 9fdccb3548
43 changed files with 952 additions and 1120 deletions
Download Patch File Download Diff File Expand all files Collapse all files

53
cdc_rsync/BUILD
View File

@@ -1,12 +1,16 @@
load(
"//tools:windows_cc_library.bzl",
"cc_windows_shared_library",
)
package(default_visibility = [
"//:__subpackages__",
])
cc_binary(
name = "cdc_rsync",
srcs = ["main.cc"],
deps = [
":cdc_rsync_client",
":params",
],
)
cc_library(
name = "client_file_info",
hdrs = ["client_file_info.h"],
@@ -57,25 +61,16 @@ cc_test(
],
)
cc_windows_shared_library(
name = "cdc_rsync",
srcs = [
"cdc_rsync.cc",
"cdc_rsync_client.cc",
"dllmain.cc",
],
hdrs = [
"cdc_rsync.h",
"cdc_rsync_client.h",
"error_messages.h",
],
cc_library(
name = "cdc_rsync_client",
srcs = ["cdc_rsync_client.cc"],
hdrs = ["cdc_rsync_client.h"],
linkopts = select({
"//tools:windows": [
"/DEFAULTLIB:Ws2_32.lib", # Sockets, e.g. recv, send, WSA*.
],
"//conditions:default": [],
}),
local_defines = ["COMPILING_DLL"],
target_compatible_with = ["@platforms//os:windows"],
deps = [
":client_socket",
@@ -128,6 +123,28 @@ cc_test(
],
)
cc_library(
name = "params",
srcs = ["params.cc"],
hdrs = ["params.h"],
deps = [
":cdc_rsync_client",
"@com_github_zstd//:zstd",
"@com_google_absl//absl/status",
],
)
cc_test(
name = "params_test",
srcs = ["params_test.cc"],
data = ["testdata/root.txt"] + glob(["testdata/params/**"]),
deps = [
":params",
"//common:test_main",
"@com_google_googletest//:gtest",
],
)
cc_library(
name = "progress_tracker",
srcs = ["progress_tracker.cc"],

125
cdc_rsync/cdc_rsync.cc
View File

@@ -1,125 +0,0 @@
// Copyright 2022 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "cdc_rsync/cdc_rsync.h"
#include <vector>
#include "cdc_rsync/cdc_rsync_client.h"
#include "cdc_rsync/error_messages.h"
#include "common/log.h"
#include "common/path_filter.h"
#include "common/status.h"
namespace cdc_ft {
namespace {
ReturnCode TagToMessage(Tag tag, const Options* options, std::string* msg) {
msg->clear();
switch (tag) {
case Tag::kSocketEof:
*msg = kMsgConnectionLost;
return ReturnCode::kConnectionLost;
case Tag::kAddressInUse:
*msg = kMsgAddressInUse;
return ReturnCode::kAddressInUse;
case Tag::kDeployServer:
*msg = kMsgDeployFailed;
return ReturnCode::kDeployFailed;
case Tag::kInstancePickerNotAvailableInQuietMode:
*msg = kMsgInstancePickerNotAvailableInQuietMode;
return ReturnCode::kInstancePickerNotAvailableInQuietMode;
case Tag::kConnectionTimeout:
*msg =
absl::StrFormat(kMsgFmtConnectionTimeout, options->ip, options->port);
return ReturnCode::kConnectionTimeout;
case Tag::kCount:
return ReturnCode::kGenericError;
}
// Should not happen (TM). Will fall back to status message in this case.
return ReturnCode::kGenericError;
}
PathFilter::Rule::Type ToInternalType(FilterRule::Type type) {
switch (type) {
case FilterRule::Type::kInclude:
return PathFilter::Rule::Type::kInclude;
case FilterRule::Type::kExclude:
return PathFilter::Rule::Type::kExclude;
}
assert(false);
return PathFilter::Rule::Type::kInclude;
}
} // namespace
ReturnCode Sync(const Options* options, const FilterRule* filter_rules,
size_t num_filter_rules, const char* sources_dir,
const char* const* sources, size_t num_sources,
const char* destination, const char** error_message) {
LogLevel log_level = Log::VerbosityToLogLevel(options->verbosity);
Log::Initialize(std::make_unique<ConsoleLog>(log_level));
PathFilter path_filter;
for (size_t n = 0; n < num_filter_rules; ++n) {
path_filter.AddRule(ToInternalType(filter_rules[n].type),
filter_rules[n].pattern);
}
std::vector<std::string> sources_vec;
for (size_t n = 0; n < num_sources; ++n) {
sources_vec.push_back(sources[n]);
}
// Run rsync.
GgpRsyncClient client(*options, std::move(path_filter), sources_dir,
std::move(sources_vec), destination);
absl::Status status = client.Run();
if (status.ok()) {
*error_message = nullptr;
return ReturnCode::kOk;
}
std::string msg;
ReturnCode code = ReturnCode::kGenericError;
absl::optional<Tag> tag = GetTag(status);
if (tag.has_value()) {
code = TagToMessage(tag.value(), options, &msg);
}
// Fall back to status message.
if (msg.empty()) {
msg = std::string(status.message());
} else if (options->verbosity >= 2) {
// In verbose mode, log the status as well, so nothing gets lost.
LOG_ERROR("%s", status.ToString().c_str());
}
// Store error message in static buffer (don't use std::string through DLL
// boundary!).
static char buf[1024] = {0};
strncpy_s(buf, msg.c_str(), _TRUNCATE);
*error_message = buf;
return code;
}
} // namespace cdc_ft

107
cdc_rsync/cdc_rsync.h
View File

@@ -1,107 +0,0 @@
/*
* Copyright 2022 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef CDC_RSYNC_CDC_RSYNC_H_
#define CDC_RSYNC_CDC_RSYNC_H_
#ifdef COMPILING_DLL
#define CDC_RSYNC_API __declspec(dllexport)
#else
#define CDC_RSYNC_API __declspec(dllimport)
#endif
namespace cdc_ft {
#ifdef __cplusplus
extern "C" {
#endif
struct Options {
const char* ip = nullptr;
int port = 0;
bool delete_ = false;
bool recursive = false;
int verbosity = 0;
bool quiet = false;
bool whole_file = false;
bool relative = false;
bool compress = false;
bool checksum = false;
bool dry_run = false;
bool existing = false;
bool json = false;
const char* copy_dest = nullptr;
int compress_level = 6;
int connection_timeout_sec = 10;
// Compression level 0 is invalid.
static constexpr int kMinCompressLevel = -5;
static constexpr int kMaxCompressLevel = 22;
};
// Rule for including/excluding files.
struct FilterRule {
enum class Type {
kInclude,
kExclude,
};
Type type;
const char* pattern;
FilterRule(Type type, const char* pattern) : type(type), pattern(pattern) {}
};
enum class ReturnCode {
// No error. Will match the tool's exit code, so OK must be 0.
kOk = 0,
// Generic error.
kGenericError = 1,
// Server connection timed out.
kConnectionTimeout = 2,
// Connection to the server was shut down unexpectedly.
kConnectionLost = 3,
// Binding to the forward port failed, probably because there's another
// instance of cdc_rsync running.
kAddressInUse = 4,
// Server deployment failed. This should be rare, it means that the server
// components were successfully copied, but the up-to-date check still fails.
kDeployFailed = 5,
// Gamelet selection asks for user input, but we are in quiet mode.
kInstancePickerNotAvailableInQuietMode = 6,
};
// Calling Sync() a second time overwrites the data in |error_message|.
CDC_RSYNC_API ReturnCode Sync(const Options* options,
const FilterRule* filter_rules,
size_t filter_num_rules, const char* sources_dir,
const char* const* sources, size_t num_sources,
const char* destination,
const char** error_message);
#ifdef __cplusplus
} // extern "C"
#endif
} // namespace cdc_ft
#endif // CDC_RSYNC_CDC_RSYNC_H_

87
cdc_rsync/cdc_rsync.vcxproj Normal file
View File

@@ -0,0 +1,87 @@
<?xml version="1.0" encoding="utf-8"?>
<Project DefaultTargets="Build" ToolsVersion="15.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<ItemGroup Label="ProjectConfigurations">
<ProjectConfiguration Include="Debug|x64">
<Configuration>Debug</Configuration>
<Platform>x64</Platform>
</ProjectConfiguration>
<ProjectConfiguration Include="Release|x64">
<Configuration>Release</Configuration>
<Platform>x64</Platform>
</ProjectConfiguration>
</ItemGroup>
<PropertyGroup Label="Globals">
<VCProjectVersion>15.0</VCProjectVersion>
<ProjectGuid>{3FAC852A-00A8-4CFB-9160-07EFF2B73562}</ProjectGuid>
<Keyword>Win32Proj</Keyword>
<RootNamespace>cdc_rsync</RootNamespace>
<WindowsTargetPlatformVersion Condition="$(VisualStudioVersion) == 15">$([Microsoft.Build.Utilities.ToolLocationHelper]::GetLatestSDKTargetPlatformVersion('Windows', '10.0'))</WindowsTargetPlatformVersion>
<WindowsTargetPlatformVersion Condition="$(VisualStudioVersion) == 16">10.0</WindowsTargetPlatformVersion>
</PropertyGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
<ConfigurationType>Makefile</ConfigurationType>
<UseDebugLibraries>true</UseDebugLibraries>
<PlatformToolset Condition="$(VisualStudioVersion) == 15">v141</PlatformToolset>
<PlatformToolset Condition="$(VisualStudioVersion) == 16">v142</PlatformToolset>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
<ConfigurationType>Makefile</ConfigurationType>
<UseDebugLibraries>false</UseDebugLibraries>
<PlatformToolset Condition="$(VisualStudioVersion) == 15">v141</PlatformToolset>
<PlatformToolset Condition="$(VisualStudioVersion) == 16">v142</PlatformToolset>
</PropertyGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
<ImportGroup Label="Shared">
<Import Project="..\all_files.vcxitems" Label="Shared" />
</ImportGroup>
<ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
</ImportGroup>
<ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
</ImportGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
<OutDir>$(SolutionDir)bazel-out\x64_windows-dbg\bin\cdc_rsync\</OutDir>
<AdditionalOptions>/std:c++17</AdditionalOptions>
<NMakePreprocessorDefinitions>UNICODE</NMakePreprocessorDefinitions>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
<OutDir>$(SolutionDir)bazel-out\x64_windows-opt\bin\cdc_rsync\</OutDir>
<NMakePreprocessorDefinitions>UNICODE</NMakePreprocessorDefinitions>
<AdditionalOptions>/std:c++17</AdditionalOptions>
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="..\cdc_rsync_server\cdc_rsync_server.vcxproj">
<Project>{4ece65e0-d950-4b96-8ad5-0313261b8c8d}</Project>
<ReferenceOutputAssembly>false</ReferenceOutputAssembly>
<LinkLibraryDependencies>false</LinkLibraryDependencies>
</ProjectReference>
</ItemGroup>
<!-- Prevent console from being closed -->
<ItemDefinitionGroup>
<Link>
<SubSystem>Console</SubSystem>
</Link>
</ItemDefinitionGroup>
<!-- Bazel setup -->
<PropertyGroup>
<BazelTargets>//cdc_rsync</BazelTargets>
<BazelOutputFile>cdc_rsync.exe</BazelOutputFile>
<BazelIncludePaths>..\;..\third_party\absl;..\third_party\blake3\c;..\bazel-stadia-file-transfer\external\com_github_zstd\lib;..\third_party\googletest\googletest\include;..\third_party\protobuf\src;$(VC_IncludePath);$(WindowsSDK_IncludePath)</BazelIncludePaths>
<BazelSourcePathPrefix>..\/</BazelSourcePathPrefix>
</PropertyGroup>
<Import Project="..\NMakeBazelProject.targets" />
<!-- For some reason, msbuild doesn't include this file, so copy it explicitly. -->
<!-- TODO: Reenable once we can cross-compile these.
<PropertyGroup>
<GgpRsyncServerFile>$(SolutionDir)bazel-out\k8-$(BazelCompilationMode)\bin\cdc_rsync_server\cdc_rsync_server</GgpRsyncServerFile>
</PropertyGroup>
<Target Name="CopyServer" Inputs="$(GgpRsyncServerFile)" Outputs="$(OutDir)cdc_rsync_server" AfterTargets="Build">
<Copy SourceFiles="$(GgpRsyncServerFile)" DestinationFiles="$(OutDir)cdc_rsync_server" />
</Target>
-->
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
<ImportGroup Label="ExtensionTargets">
</ImportGroup>
</Project>

2
cdc_rsync/cdc_rsync.vcxproj.filters Normal file
View File

@@ -0,0 +1,2 @@
<?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003" />

127
cdc_rsync/cdc_rsync_client.cc
View File

@@ -47,7 +47,7 @@ constexpr int kExitCodeNotFound = 127;
constexpr int kForwardPortFirst = 44450;
constexpr int kForwardPortLast = 44459;
constexpr char kGgpServerFilename[] = "cdc_rsync_server";
constexpr char kRemoteToolsBinDir[] = "/opt/developer/tools/bin/";
constexpr char kRemoteToolsBinDir[] = "~/.cache/cdc_file_transfer/";
SetOptionsRequest::FilterRule::Type ToProtoType(PathFilter::Rule::Type type) {
switch (type) {
@@ -94,14 +94,12 @@ absl::Status GetServerExitStatus(int exit_code, const std::string& error_msg) {
} // namespace
GgpRsyncClient::GgpRsyncClient(const Options& options, PathFilter path_filter,
std::string sources_dir,
CdcRsyncClient::CdcRsyncClient(const Options& options,
std::vector<std::string> sources,
std::string destination)
std::string user_host, std::string destination)
: options_(options),
path_filter_(std::move(path_filter)),
sources_dir_(std::move(sources_dir)),
sources_(std::move(sources)),
user_host_(std::move(user_host)),
destination_(std::move(destination)),
remote_util_(options.verbosity, options.quiet, &process_factory_,
/*forward_output_to_log=*/false),
@@ -109,24 +107,26 @@ GgpRsyncClient::GgpRsyncClient(const Options& options, PathFilter path_filter,
kForwardPortFirst, kForwardPortLast, &process_factory_,
&remote_util_),
printer_(options.quiet, Util::IsTTY() && !options.json),
progress_(&printer_, options.verbosity, options.json) {}
progress_(&printer_, options.verbosity, options.json) {
if (!options_.ssh_command.empty()) {
remote_util_.SetSshCommand(options_.ssh_command);
}
if (!options_.scp_command.empty()) {
remote_util_.SetScpCommand(options_.scp_command);
}
}
GgpRsyncClient::~GgpRsyncClient() {
CdcRsyncClient::~CdcRsyncClient() {
message_pump_.StopMessagePump();
socket_.Disconnect();
}
absl::Status GgpRsyncClient::Run() {
absl::Status status = remote_util_.GetInitStatus();
if (!status.ok()) {
return WrapStatus(status, "Failed to initialize critical components");
}
absl::Status CdcRsyncClient::Run() {
// Initialize |remote_util_|.
remote_util_.SetIpAndPort(options_.ip, options_.port);
remote_util_.SetUserHostAndPort(user_host_, options_.port);
// Start the server process.
status = StartServer();
absl::Status status = StartServer();
if (HasTag(status, Tag::kDeployServer)) {
// Gamelet components are not deployed or out-dated. Deploy and retry.
status = DeployServer();
@@ -166,7 +166,7 @@ absl::Status GgpRsyncClient::Run() {
return status;
}
absl::Status GgpRsyncClient::StartServer() {
absl::Status CdcRsyncClient::StartServer() {
assert(!server_process_);
// Components are expected to reside in the same dir as the executable.
@@ -187,8 +187,8 @@ absl::Status GgpRsyncClient::StartServer() {
std::string component_args = GameletComponent::ToCommandLineArgs(components);
// Find available local and remote ports for port forwarding.
absl::StatusOr<int> port_res =
port_manager_.ReservePort(options_.connection_timeout_sec);
absl::StatusOr<int> port_res = port_manager_.ReservePort(
/*check_remote=*/false, /*remote_timeout_sec unused*/ 0);
constexpr char kErrorMsg[] = "Failed to find available port";
if (absl::IsDeadlineExceeded(port_res.status())) {
// Server didn't respond in time.
@@ -205,9 +205,11 @@ absl::Status GgpRsyncClient::StartServer() {
std::string(kRemoteToolsBinDir) + kGgpServerFilename;
// Test existence manually to prevent misleading bash output message
// "bash: .../cdc_rsync_server: No such file or directory".
std::string remote_command = absl::StrFormat(
"if [ ! -f %s ]; then exit %i; fi; %s %i %s", remote_server_path,
kExitCodeNotFound, remote_server_path, port, component_args);
// Also create the bin dir because otherwise scp below might fail.
std::string remote_command =
absl::StrFormat("mkdir -p %s; if [ ! -f %s ]; then exit %i; fi; %s %i %s",
kRemoteToolsBinDir, remote_server_path, kExitCodeNotFound,
remote_server_path, port, component_args);
ProcessStartInfo start_info =
remote_util_.BuildProcessStartInfoForSshPortForwardAndCommand(
port, port, false, remote_command);
@@ -225,16 +227,25 @@ absl::Status GgpRsyncClient::StartServer() {
}
// Wait until the server process is listening.
auto detect_listening = [is_listening = &is_server_listening_]() -> bool {
return *is_listening;
Stopwatch timeout_timer;
bool is_timeout = false;
auto detect_listening_or_timeout = [is_listening = &is_server_listening_,
timeout = options_.connection_timeout_sec,
&timeout_timer, &is_timeout]() -> bool {
is_timeout = timeout_timer.ElapsedSeconds() > timeout;
return *is_listening || is_timeout;
};
status = process->RunUntil(detect_listening);
status = process->RunUntil(detect_listening_or_timeout);
if (!status.ok()) {
// Some internal process error. Note that this does NOT mean that
// cdc_rsync_server does not exist. In that case, the ssh process exits with
// code 127.
return status;
}
if (is_timeout) {
return SetTag(absl::DeadlineExceededError("Timeout while starting server"),
Tag::kConnectionTimeout);
}
if (process->HasExited()) {
// Don't re-deploy for code > kServerExitCodeOutOfDate, which means that the
@@ -263,7 +274,7 @@ absl::Status GgpRsyncClient::StartServer() {
return absl::OkStatus();
}
absl::Status GgpRsyncClient::StopServer() {
absl::Status CdcRsyncClient::StopServer() {
assert(server_process_);
// Close socket.
@@ -282,7 +293,7 @@ absl::Status GgpRsyncClient::StopServer() {
return absl::OkStatus();
}
absl::Status GgpRsyncClient::HandleServerOutput(const char* data) {
absl::Status CdcRsyncClient::HandleServerOutput(const char* data) {
// Note: This is called from a background thread!
// Handle server error messages. Unfortunately, if the server prints to
@@ -319,7 +330,7 @@ absl::Status GgpRsyncClient::HandleServerOutput(const char* data) {
return absl::OkStatus();
}
absl::Status GgpRsyncClient::Sync() {
absl::Status CdcRsyncClient::Sync() {
absl::Status status = SendOptions();
if (!status.ok()) {
return WrapStatus(status, "Failed to send options to server");
@@ -377,7 +388,7 @@ absl::Status GgpRsyncClient::Sync() {
return status;
}
absl::Status GgpRsyncClient::DeployServer() {
absl::Status CdcRsyncClient::DeployServer() {
assert(!server_process_);
std::string exe_dir;
@@ -409,34 +420,26 @@ absl::Status GgpRsyncClient::DeployServer() {
return WrapStatus(status, "Failed to copy cdc_rsync_server to instance");
}
// Make cdc_rsync_server executable.
status = remote_util_.Chmod("a+x", remoteServerTmpPath);
// Do 3 things in one SSH command, to save time:
// - Make the old cdc_rsync_server writable (if it exists).
// - Make the new cdc_rsync_server executable.
// - Replace the old cdc_rsync_server by the new one.
std::string old_path = RemoteUtil::EscapeForWindows(
std::string(kRemoteToolsBinDir) + kGgpServerFilename);
std::string new_path = RemoteUtil::EscapeForWindows(remoteServerTmpPath);
std::string replace_cmd = absl::StrFormat(
" ([ ! -f %s ] || chmod u+w %s) && chmod a+x %s && mv %s %s", old_path,
old_path, new_path, new_path, old_path);
status = remote_util_.Run(replace_cmd, "chmod && chmod && mv");
if (!status.ok()) {
return WrapStatus(status,
"Failed to set executable flag on cdc_rsync_server");
}
// Make old file writable. Mv might fail to overwrite it, e.g. if someone made
// it read-only.
std::string remoteServerPath =
std::string(kRemoteToolsBinDir) + kGgpServerFilename;
status = remote_util_.Chmod("u+w", remoteServerPath, /*quiet=*/true);
if (!status.ok()) {
LOG_DEBUG("chmod u+w %s failed (expected if file does not exist): %s",
remoteServerPath, status.ToString());
}
// Replace old file by new file.
status = remote_util_.Mv(remoteServerTmpPath, remoteServerPath);
if (!status.ok()) {
return WrapStatus(status, "Failed to replace '%s' by '%s'",
remoteServerPath, remoteServerTmpPath);
"Failed to replace old cdc_rsync_server by new one");
}
return absl::OkStatus();
}
absl::Status GgpRsyncClient::SendOptions() {
absl::Status CdcRsyncClient::SendOptions() {
LOG_INFO("Sending options");
SetOptionsRequest request;
@@ -448,7 +451,7 @@ absl::Status GgpRsyncClient::SendOptions() {
request.set_compress(options_.compress);
request.set_relative(options_.relative);
for (const PathFilter::Rule& rule : path_filter_.GetRules()) {
for (const PathFilter::Rule& rule : options_.filter.GetRules()) {
SetOptionsRequest::FilterRule* filter_rule = request.add_filter_rules();
filter_rule->set_type(ToProtoType(rule.type));
filter_rule->set_pattern(rule.pattern);
@@ -457,7 +460,7 @@ absl::Status GgpRsyncClient::SendOptions() {
request.set_checksum(options_.checksum);
request.set_dry_run(options_.dry_run);
request.set_existing(options_.existing);
if (options_.copy_dest) {
if (!options_.copy_dest.empty()) {
request.set_copy_dest(options_.copy_dest);
}
@@ -470,13 +473,13 @@ absl::Status GgpRsyncClient::SendOptions() {
return absl::OkStatus();
}
absl::Status GgpRsyncClient::FindAndSendAllSourceFiles() {
absl::Status CdcRsyncClient::FindAndSendAllSourceFiles() {
LOG_INFO("Finding and sending all sources files");
Stopwatch stopwatch;
FileFinderAndSender file_finder(&path_filter_, &message_pump_, &progress_,
sources_dir_, options_.recursive,
FileFinderAndSender file_finder(&options_.filter, &message_pump_, &progress_,
options_.sources_dir, options_.recursive,
options_.relative);
progress_.StartFindFiles();
@@ -497,7 +500,7 @@ absl::Status GgpRsyncClient::FindAndSendAllSourceFiles() {
return absl::OkStatus();
}
absl::Status GgpRsyncClient::ReceiveFileStats() {
absl::Status CdcRsyncClient::ReceiveFileStats() {
LOG_INFO("Receiving file stats");
SendFileStatsResponse response;
@@ -517,7 +520,7 @@ absl::Status GgpRsyncClient::ReceiveFileStats() {
return absl::OkStatus();
}
absl::Status GgpRsyncClient::ReceiveDeletedFiles() {
absl::Status CdcRsyncClient::ReceiveDeletedFiles() {
LOG_INFO("Receiving path of deleted files");
std::string current_directory;
@@ -548,7 +551,7 @@ absl::Status GgpRsyncClient::ReceiveDeletedFiles() {
return absl::OkStatus();
}
absl::Status GgpRsyncClient::ReceiveFileIndices(
absl::Status CdcRsyncClient::ReceiveFileIndices(
const char* file_type, std::vector<uint32_t>* file_indices) {
LOG_INFO("Receiving indices of %s files", file_type);
@@ -582,7 +585,7 @@ absl::Status GgpRsyncClient::ReceiveFileIndices(
return absl::OkStatus();
}
absl::Status GgpRsyncClient::SendMissingFiles() {
absl::Status CdcRsyncClient::SendMissingFiles() {
if (missing_file_indices_.empty()) {
return absl::OkStatus();
}
@@ -653,7 +656,7 @@ absl::Status GgpRsyncClient::SendMissingFiles() {
return absl::OkStatus();
}
absl::Status GgpRsyncClient::ReceiveSignaturesAndSendDelta() {
absl::Status CdcRsyncClient::ReceiveSignaturesAndSendDelta() {
if (changed_file_indices_.empty()) {
return absl::OkStatus();
}
@@ -731,7 +734,7 @@ absl::Status GgpRsyncClient::ReceiveSignaturesAndSendDelta() {
return absl::OkStatus();
}
absl::Status GgpRsyncClient::StartCompressionStream() {
absl::Status CdcRsyncClient::StartCompressionStream() {
assert(!compression_stream_);
// Notify server that data is compressed from now on.
@@ -762,7 +765,7 @@ absl::Status GgpRsyncClient::StartCompressionStream() {
return absl::OkStatus();
}
absl::Status GgpRsyncClient::StopCompressionStream() {
absl::Status CdcRsyncClient::StopCompressionStream() {
assert(compression_stream_);
// Finish writing to |compression_process_|'s stdin and change back to

40
cdc_rsync/cdc_rsync_client.h
View File

@@ -22,7 +22,6 @@
#include "absl/status/status.h"
#include "cdc_rsync/base/message_pump.h"
#include "cdc_rsync/cdc_rsync.h"
#include "cdc_rsync/client_socket.h"
#include "cdc_rsync/progress_tracker.h"
#include "common/path_filter.h"
@@ -34,13 +33,38 @@ namespace cdc_ft {
class Process;
class ZstdStream;
class GgpRsyncClient {
class CdcRsyncClient {
public:
GgpRsyncClient(const Options& options, PathFilter filter,
std::string sources_dir, std::vector<std::string> sources,
std::string destination);
struct Options {
int port = RemoteUtil::kDefaultSshPort;
bool delete_ = false;
bool recursive = false;
int verbosity = 0;
bool quiet = false;
bool whole_file = false;
bool relative = false;
bool compress = false;
bool checksum = false;
bool dry_run = false;
bool existing = false;
bool json = false;
std::string copy_dest;
int compress_level = 6;
int connection_timeout_sec = 10;
std::string ssh_command;
std::string scp_command;
std::string sources_dir; // Base dir for files loaded for --files-from.
PathFilter filter;
~GgpRsyncClient();
// Compression level 0 is invalid.
static constexpr int kMinCompressLevel = -5;
static constexpr int kMaxCompressLevel = 22;
};
CdcRsyncClient(const Options& options, std::vector<std::string> sources,
std::string user_host, std::string destination);
~CdcRsyncClient();
// Deploys the server if necessary, starts it and runs the rsync procedure.
absl::Status Run();
@@ -93,11 +117,9 @@ class GgpRsyncClient {
absl::Status StopCompressionStream();
Options options_;
PathFilter path_filter_;
const std::string sources_dir_;
std::vector<std::string> sources_;
const std::string user_host_;
const std::string destination_;
WinProcessFactory process_factory_;
RemoteUtil remote_util_;
PortManager port_manager_;

29
cdc_rsync/dllmain.cc
View File

@@ -1,29 +0,0 @@
// Copyright 2022 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#define WIN32_LEAN_AND_MEAN
#include <windows.h>
BOOL APIENTRY DllMain(HMODULE /* hModule */, DWORD ul_reason_for_call,
LPVOID /* lpReserved */
) {
switch (ul_reason_for_call) {
case DLL_PROCESS_ATTACH:
case DLL_THREAD_ATTACH:
case DLL_THREAD_DETACH:
case DLL_PROCESS_DETACH:
break;
}
return TRUE;
}

54
cdc_rsync/error_messages.h
View File

@@ -1,54 +0,0 @@
/*
* Copyright 2022 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef CDC_RSYNC_ERROR_MESSAGES_H_
#define CDC_RSYNC_ERROR_MESSAGES_H_
namespace cdc_ft {
// Server connection timed out. SSH probably stale.
constexpr char kMsgFmtConnectionTimeout[] =
"Server connection timed out. Please re-run 'ggp ssh init' and verify that "
"the IP '%s' and the port '%i' are correct.";
// Server connection timed out and IP was not passed in. Probably network error.
constexpr char kMsgConnectionTimeoutWithIp[] =
"Server connection timed out. Please check your network connection.";
// Receiving pipe end was shut down unexpectedly.
constexpr char kMsgConnectionLost[] =
"The connection to the instance was shut down unexpectedly.";
// Binding to the port failed.
constexpr char kMsgAddressInUse[] =
"Failed to establish a connection to the instance. All ports are already "
"in use. This can happen if another instance of this command is running. "
"Currently, only 10 simultaneous connections are supported.";
// Deployment failed even though gamelet components were copied successfully.
constexpr char kMsgDeployFailed[] =
"Failed to deploy the instance components for unknown reasons. "
"Please report this issue.";
// Picking an instance is not allowed in quiet mode.
constexpr char kMsgInstancePickerNotAvailableInQuietMode[] =
"Multiple gamelet instances are reserved, but the instance picker is not "
"available in quiet mode. Please specify --instance or remove -q resp. "
"--quiet.";
} // namespace cdc_ft
#endif // CDC_RSYNC_ERROR_MESSAGES_H_

147
cdc_rsync/main.cc Normal file
View File

@@ -0,0 +1,147 @@
// Copyright 2022 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#define WIN32_LEAN_AND_MEAN
#include <windows.h>
#include <string>
#include <vector>
#include "cdc_rsync/cdc_rsync_client.h"
#include "cdc_rsync/params.h"
#include "common/log.h"
#include "common/status.h"
#include "common/util.h"
namespace {
enum class ReturnCode {
// No error. Will match the tool's exit code, so OK must be 0.
kOk = 0,
// Generic error.
kGenericError = 1,
// Server connection timed out.
kConnectionTimeout = 2,
// Connection to the server was shut down unexpectedly.
kConnectionLost = 3,
// Binding to the forward port failed, probably because there's another
// instance of cdc_rsync running.
kAddressInUse = 4,
// Server deployment failed. This should be rare, it means that the server
// components were successfully copied, but the up-to-date check still fails.
kDeployFailed = 5,
};
ReturnCode TagToMessage(cdc_ft::Tag tag,
const cdc_ft::params::Parameters& params,
std::string* msg) {
msg->clear();
switch (tag) {
case cdc_ft::Tag::kSocketEof:
// Receiving pipe end was shut down unexpectedly.
*msg = "The connection to the instance was shut down unexpectedly.";
return ReturnCode::kConnectionLost;
case cdc_ft::Tag::kAddressInUse:
*msg =
"Failed to establish a connection to the instance. All ports are "
"already in use. This can happen if another instance of this command "
"is running. Currently, only 10 simultaneous connections are "
"supported.";
return ReturnCode::kAddressInUse;
case cdc_ft::Tag::kDeployServer:
*msg =
"Failed to deploy the instance components for unknown reasons. "
"Please report this issue.";
return ReturnCode::kDeployFailed;
case cdc_ft::Tag::kConnectionTimeout:
// Server connection timed out. SSH probably stale.
*msg = absl::StrFormat(
"Server connection timed out. Verify that host '%s' and port '%i' "
"are correct, or specify a larger timeout with --contimeout.",
params.user_host, params.options.port);
return ReturnCode::kConnectionTimeout;
case cdc_ft::Tag::kCount:
return ReturnCode::kGenericError;
}
// Should not happen (TM). Will fall back to status message in this case.
return ReturnCode::kGenericError;
}
} // namespace
int wmain(int argc, wchar_t* argv[]) {
// Convert args from wide to UTF8 strings.
std::vector<std::string> utf8_str_args;
utf8_str_args.reserve(argc);
for (int i = 0; i < argc; i++) {
utf8_str_args.push_back(cdc_ft::Util::WideToUtf8Str(argv[i]));
}
// Convert args from UTF8 strings to UTF8 c-strings.
std::vector<const char*> utf8_args;
utf8_args.reserve(argc);
for (const auto& utf8_str_arg : utf8_str_args) {
utf8_args.push_back(utf8_str_arg.c_str());
}
// Read parameters from the environment and the command line.
cdc_ft::params::Parameters parameters;
if (!cdc_ft::params::Parse(argc, utf8_args.data(), &parameters)) {
return 1;
}
// Initialize logging.
cdc_ft::LogLevel log_level =
cdc_ft::Log::VerbosityToLogLevel(parameters.options.verbosity);
cdc_ft::Log::Initialize(std::make_unique<cdc_ft::ConsoleLog>(log_level));
// Run rsync.
cdc_ft::CdcRsyncClient client(parameters.options, parameters.sources,
parameters.user_host, parameters.destination);
absl::Status status = client.Run();
if (status.ok()) {
return static_cast<int>(ReturnCode::kOk);
}
// Get an error message from the tag associated with the status.
std::string error_message;
ReturnCode code = ReturnCode::kGenericError;
absl::optional<cdc_ft::Tag> tag = cdc_ft::GetTag(status);
if (tag.has_value()) {
code = TagToMessage(tag.value(), parameters, &error_message);
}
// Fall back to status message if there was no tag.
if (error_message.empty()) {
error_message = status.message();
} else if (parameters.options.verbosity >= 2) {
// In verbose mode, log the status as well, so nothing gets lost.
LOG_ERROR("%s", status.ToString());
}
if (!error_message.empty()) {
fprintf(stderr, "Error: %s\n", error_message.c_str());
}
return static_cast<int>(code);
}

493
cdc_rsync/params.cc Normal file
View File

@@ -0,0 +1,493 @@
// Copyright 2022 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "cdc_rsync/params.h"
#include <cassert>
#include "absl/status/status.h"
#include "absl/strings/str_format.h"
#include "absl/strings/str_split.h"
#include "common/path.h"
#include "lib/zstd.h"
namespace cdc_ft {
namespace params {
namespace {
using Options = CdcRsyncClient::Options;
template <typename... Args>
void PrintError(const absl::FormatSpec<Args...>& format, Args... args) {
std::cerr << "Error: " << absl::StrFormat(format, args...) << std::endl;
}
enum class OptionResult { kConsumedKey, kConsumedKeyValue, kError };
const char kHelpText[] =
R"(Copy local files to a gamelet
Synchronizes local files and files on a gamelet. Matching files are skipped.
For partially matching files only the deltas are transferred.
Usage:
cdc_rsync [options] source [source]... [user@]host:destination
Parameters:
source Local file or directory to be copied
user Remote SSH user name
host Remote host or IP address
destination Remote destination directory
Options:
--ip string Gamelet IP. Required.
--port number SSH port to use. Required.
--contimeout sec Gamelet connection timeout in seconds (default: 10)
-q, --quiet Quiet mode, only print errors
-v, --verbose Increase output verbosity
--json Print JSON progress
-n, --dry-run Perform a trial run with no changes made
-r, --recursive Recurse into directories
--delete Delete extraneous files from destination directory
-z, --compress Compress file data during the transfer
--compress-level num Explicitly set compression level (default: 6)
-c, --checksum Skip files based on checksum, not mod-time & size
-W, --whole-file Always copy files whole,
do not apply delta-transfer algorithm
--exclude pattern Exclude files matching pattern
--exclude-from file Read exclude patterns from file
--include pattern Don't exclude files matching pattern
--include-from file Read include patterns from file
--files-from file Read list of source files from file
-R, --relative Use relative path names
--existing Skip creating new files on instance
--copy-dest dir Use files from dir as sync base if files are missing
--ssh-command Path and arguments of SSH command to use, e.g.
C:\path\to\ssh.exe -F config -i id_rsa -oStrictHostKeyChecking=yes -oUserKnownHostsFile="""known_hosts"""
Can also be specified by the CDC_SSH_COMMAND environment variable.
--scp-command Path and arguments of SSH command to use, e.g.
C:\path\to\scp.exe -F config -i id_rsa -oStrictHostKeyChecking=yes -oUserKnownHostsFile="""known_hosts"""
Can also be specified by the CDC_SCP_COMMAND environment variable.
-h --help Help for cdc_rsync
)";
constexpr char kSshCommandEnvVar[] = "CDC_SSH_COMMAND";
constexpr char kScpCommandEnvVar[] = "CDC_SCP_COMMAND";
// Populates some parameters from environment variables.
void PopulateFromEnvVars(Parameters* parameters) {
path::GetEnv(kSshCommandEnvVar, &parameters->options.ssh_command)
.IgnoreError();
path::GetEnv(kScpCommandEnvVar, &parameters->options.scp_command)
.IgnoreError();
}
// Handles the --exclude-from and --include-from options.
OptionResult HandleFilterRuleFile(const std::string& option_name,
const char* path, PathFilter::Rule::Type type,
Parameters* params) {
if (!path) {
PrintError("Option '%s' needs a value", option_name);
return OptionResult::kError;
}
std::vector<std::string> patterns;
absl::Status status = path::ReadAllLines(
path, &patterns,
path::ReadFlags::kRemoveEmpty | path::ReadFlags::kTrimWhitespace);
if (!status.ok()) {
PrintError("Failed to read file '%s' for %s option: %s", path, option_name,
status.message());
return OptionResult::kError;
}
for (std::string& pattern : patterns) {
params->options.filter.AddRule(type, std::move(pattern));
}
return OptionResult::kConsumedKeyValue;
}
// Loads sources for --files-from option. |sources| must contain at most one
// path and that path must be an existing directory. This directory is returned
// in |sources_dir|. The method then loads all sources line-by-line from
// |sources_file| and stores them into |sources|.
bool LoadFilesFrom(const std::string& files_from,
std::vector<std::string>* sources,
std::string* sources_dir) {
if (sources->size() > 1) {
PrintError(
"Expected at most 1 source for the --files-from option, but %u "
"provided",
sources->size());
return false;
}
if (sources->size() == 1 && !path::DirExists(sources->at(0))) {
PrintError(
"The source '%s' must be an existing directory for the --files-from "
"option",
sources->at(0));
return false;
}
*sources_dir = sources->empty() ? std::string() : sources->at(0);
if (!sources_dir->empty()) {
path::EnsureEndsWithPathSeparator(sources_dir);
}
sources->clear();
absl::Status status = path::ReadAllLines(
files_from, sources,
path::ReadFlags::kRemoveEmpty | path::ReadFlags::kTrimWhitespace);
if (!status.ok()) {
PrintError("Failed to read sources file '%s' for files-from option: %s",
files_from, status.message());
return false;
}
if (sources->empty()) {
PrintError("The file '%s' specified in the --files-from option is empty",
files_from);
}
return true;
}
OptionResult HandleParameter(const std::string& key, const char* value,
Parameters* params, bool* help) {
if (key == "port") {
if (value) {
params->options.port = atoi(value);
}
return OptionResult::kConsumedKeyValue;
}
if (key == "delete") {
params->options.delete_ = true;
return OptionResult::kConsumedKey;
}
if (key == "r" || key == "recursive") {
params->options.recursive = true;
return OptionResult::kConsumedKey;
}
if (key == "v" || key == "verbosity") {
params->options.verbosity++;
return OptionResult::kConsumedKey;
}
if (key == "q" || key == "quiet") {
params->options.quiet = true;
return OptionResult::kConsumedKey;
}
if (key == "W" || key == "whole-file") {
params->options.whole_file = true;
return OptionResult::kConsumedKey;
}
if (key == "include") {
params->options.filter.AddRule(PathFilter::Rule::Type::kInclude, value);
return OptionResult::kConsumedKeyValue;
}
if (key == "include-from") {
return HandleFilterRuleFile(key, value, PathFilter::Rule::Type::kInclude,
params);
}
if (key == "exclude") {
params->options.filter.AddRule(PathFilter::Rule::Type::kExclude, value);
return OptionResult::kConsumedKeyValue;
}
if (key == "exclude-from") {
return HandleFilterRuleFile(key, value, PathFilter::Rule::Type::kExclude,
params);
}
if (key == "files-from") {
// Implies -R.
params->options.relative = true;
params->files_from = value ? value : std::string();
return OptionResult::kConsumedKeyValue;
}
if (key == "R" || key == "relative") {
params->options.relative = true;
return OptionResult::kConsumedKey;
}
if (key == "z" || key == "compress") {
params->options.compress = true;
return OptionResult::kConsumedKey;
}
if (key == "compress-level") {
if (value) {
params->options.compress_level = atoi(value);
}
return OptionResult::kConsumedKeyValue;
}
if (key == "contimeout") {
if (value) {
params->options.connection_timeout_sec = atoi(value);
}
return OptionResult::kConsumedKeyValue;
}
if (key == "h" || key == "help") {
*help = true;
return OptionResult::kConsumedKey;
}
if (key == "c" || key == "checksum") {
params->options.checksum = true;
return OptionResult::kConsumedKey;
}
if (key == "n" || key == "dry-run") {
params->options.dry_run = true;
return OptionResult::kConsumedKey;
}
if (key == "existing") {
params->options.existing = true;
return OptionResult::kConsumedKey;
}
if (key == "copy-dest") {
params->options.copy_dest = value ? value : std::string();
return OptionResult::kConsumedKeyValue;
}
if (key == "json") {
params->options.json = true;
return OptionResult::kConsumedKey;
}
if (key == "ssh-command") {
params->options.ssh_command = value ? value : std::string();
return OptionResult::kConsumedKeyValue;
}
if (key == "scp-command") {
params->options.scp_command = value ? value : std::string();
return OptionResult::kConsumedKeyValue;
}
PrintError("Unknown option: '%s'", key);
return OptionResult::kError;
}
bool ValidateParameters(const Parameters& params, bool help) {
if (help) {
std::cout << kHelpText;
return false;
}
if (params.options.delete_ && !params.options.recursive) {
PrintError("--delete does not work without --recursive (-r).");
return false;
}
if (params.options.port <= 0 || params.options.port > UINT16_MAX) {
PrintError("--port must specify a valid port");
return false;
}
// Note: ZSTD_minCLevel() is ridiculously small (-131072), so use a
// reasonable value.
assert(ZSTD_minCLevel() <= Options::kMinCompressLevel);
assert(ZSTD_maxCLevel() == Options::kMaxCompressLevel);
static_assert(Options::kMinCompressLevel < 0);
static_assert(Options::kMaxCompressLevel > 0);
if (params.options.compress_level < Options::kMinCompressLevel ||
params.options.compress_level > Options::kMaxCompressLevel ||
params.options.compress_level == 0) {
PrintError("--compress_level must be between %i..-1 or 1..%i",
Options::kMinCompressLevel, Options::kMaxCompressLevel);
return false;
}
// Warn that any include rules not followed by an exclude rule are pointless
// as the files would be included, anyway.
const std::vector<PathFilter::Rule>& rules = params.options.filter.GetRules();
for (int n = static_cast<int>(rules.size()) - 1; n >= 0; --n) {
const PathFilter::Rule& rule = rules[n];
if (rule.type == PathFilter::Rule::Type::kExclude) {
break;
}
std::cout << "Warning: Include pattern '" << rule.pattern
<< "' has no effect, not followed by exclude pattern"
<< std::endl;
}
if (params.sources.empty() && params.destination.empty()) {
PrintError("Missing source and destination");
return false;
}
if (params.destination.empty()) {
PrintError("Missing destination");
return false;
}
if (params.sources.empty()) {
// If one arg was passed on the command line, it is not clear whether it
// was supposed to be a source or destination.
PrintError("Missing source or destination");
return false;
}
if (params.user_host.empty()) {
PrintError(
"No remote host specified in destination '%s'. "
"Expected [user@]host:destination.",
params.destination);
return false;
}
return true;
}
bool CheckOptionResult(OptionResult result, const std::string& name,
const char* value) {
switch (result) {
case OptionResult::kConsumedKey:
return true;
case OptionResult::kConsumedKeyValue:
if (!value) {
PrintError("Option '%s' needs a value", name);
return false;
}
return true;
case OptionResult::kError:
// Error message was already printed.
return false;
}
return true;
}
// Removes the user/host part of |destination| and puts it into |user_host|,
// e.g. if |destination| is initially "user@foo.com:~/file", it is "~/file"
// afterward and |user_host| is |user@foo.com|. Does not touch Windows drives,
// e.g. C:\foo.
void PopUserHost(std::string* destination, std::string* user_host) {
std::vector<std::string> parts =
absl::StrSplit(*destination, absl::MaxSplits(':', 1));
if (parts.size() < 2) return;
// Don't mistake the C part of C:\foo as user/host.
if (parts[0].size() == 1 && toupper(parts[0][0]) >= 'A' &&
toupper(parts[0][0]) <= 'Z') {
return;
}
*user_host = parts[0];
*destination = parts[1];
}
} // namespace
const char* HelpText() { return kHelpText; }
// Note that abseil has a flags library, but the C++ version doesn't support
// short names ("-q"), see https://abseil.io/docs/cpp/guides/flags. However, we
// aim to be roughly compatible with vanilla rsync, which does have short flag
// names like "-q".
bool Parse(int argc, const char* const* argv, Parameters* parameters) {
if (argc <= 1) {
std::cout << kHelpText;
return false;
}
// Before applying args, populate parameters from env vars.
PopulateFromEnvVars(parameters);
bool help = false;
for (int index = 1; index < argc; ++index) {
// Handle '--key [value]' and '--key=value' options.
bool equality_used = false;
if (strncmp(argv[index], "--", 2) == 0) {
std::string key(argv[index] + 2);
const char* value = nullptr;
size_t equality_pos = key.find("=");
if (equality_pos != std::string::npos) {
if (equality_pos + 1 < key.size()) {
value = argv[index] + 2 + equality_pos + 1;
}
key = key.substr(0, equality_pos);
equality_used = true;
} else {
value = index + 1 < argc && argv[index + 1][0] != '-' ? argv[index + 1]
: nullptr;
}
OptionResult result = HandleParameter(key, value, parameters, &help);
if (!CheckOptionResult(result, key, value)) {
return false;
}
if (!equality_used && result == OptionResult::kConsumedKeyValue) {
++index;
}
continue;
}
// Handle '-abc' options.
if (strncmp(argv[index], "-", 1) == 0) {
char key[] = "x";
char name[] = "-x";
for (const char* c = argv[index] + 1; *c != 0; ++c) {
key[0] = *c;
name[1] = *c;
OptionResult result = HandleParameter(key, nullptr, parameters, &help);
// These args shouldn't try to consume values.
assert(result != OptionResult::kConsumedKeyValue);
if (!CheckOptionResult(result, name, nullptr)) {
return false;
}
}
continue;
}
// The last added option is the destination. Move previously added options
// to the sources.
if (!parameters->destination.empty()) {
parameters->sources.push_back(std::move(parameters->destination));
}
parameters->destination = argv[index];
}
// Load files-from file (can't do it when --files-from is handled since not
// all sources might have been read at that point.
if (!parameters->files_from.empty() &&
!LoadFilesFrom(parameters->files_from, &parameters->sources,
&parameters->options.sources_dir)) {
return false;
}
PopUserHost(&parameters->destination, &parameters->user_host);
if (!ValidateParameters(*parameters, help)) {
return false;
}
return true;
}
} // namespace params
} // namespace cdc_ft

44
cdc_rsync/params.h Normal file
View File

@@ -0,0 +1,44 @@
/*
* Copyright 2022 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef CDC_RSYNC_PARAMS_H_
#define CDC_RSYNC_PARAMS_H_
#include <string>
#include <vector>
#include "cdc_rsync/cdc_rsync_client.h"
namespace cdc_ft {
namespace params {
// All cdc_rsync command line parameters.
struct Parameters {
CdcRsyncClient::Options options;
std::vector<std::string> sources;
std::string user_host;
std::string destination;
std::string files_from;
};
// Parses sources, destination and options from the command line args.
// Prints a help text if not enough arguments were given or -h/--help was given.
bool Parse(int argc, const char* const* argv, Parameters* parameters);
} // namespace params
} // namespace cdc_ft
#endif // CDC_RSYNC_PARAMS_H_

551
cdc_rsync/params_test.cc Normal file
View File

@@ -0,0 +1,551 @@
// Copyright 2022 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "cdc_rsync/params.h"
#include "absl/strings/match.h"
#include "common/log.h"
#include "common/path.h"
#include "common/status_test_macros.h"
#include "common/test_main.h"
#include "gtest/gtest.h"
namespace cdc_ft {
namespace params {
namespace {
using Options = CdcRsyncClient::Options;
constexpr char kSrc[] = "source";
constexpr char kUserHostDst[] = "user@host:destination";
constexpr char kUserHost[] = "user@host";
constexpr char kDst[] = "destination";
class TestLog : public Log {
public:
explicit TestLog() : Log(LogLevel::kInfo) {}
protected:
void WriteLogMessage(LogLevel level, const char* file, int line,
const char* func, const char* message) override {
errors_ += message;
}
private:
std::string errors_;
};
std::string NeedsValueError(const char* option_name) {
return absl::StrFormat("Option '%s' needs a value", option_name);
}
class ParamsTest : public ::testing::Test {
public:
void SetUp() override {
prev_stdout_ = std::cout.rdbuf(output_.rdbuf());
prev_stderr_ = std::cerr.rdbuf(errors_.rdbuf());
}
void TearDown() override {
std::cout.rdbuf(prev_stdout_);
std::cerr.rdbuf(prev_stderr_);
}
protected:
void ExpectNoError() const {
EXPECT_TRUE(errors_.str().empty())
<< "Expected empty stderr but got\n'" << errors_.str() << "'";
}
void ExpectOutput(const std::string& expected) const {
EXPECT_TRUE(absl::StrContains(output_.str(), expected))
<< "Expected stdout to contain '" << expected << "' but got\n'"
<< output_.str() << "'";
}
void ExpectError(const std::string& expected) const {
EXPECT_TRUE(absl::StrContains(errors_.str(), expected))
<< "Expected stderr to contain '" << expected << "' but got\n'"
<< errors_.str() << "'";
}
void ClearErrors() { errors_.str(std::string()); }
std::string base_dir_ = GetTestDataDir("params");
std::string sources_file_ = path::Join(base_dir_, "source_files.txt");
std::string empty_sources_file_ =
path::Join(base_dir_, "empty_source_files.txt");
Parameters parameters_;
std::stringstream output_;
std::stringstream errors_;
std::streambuf* prev_stdout_;
std::streambuf* prev_stderr_;
};
TEST_F(ParamsTest, ParseSucceedsDefaults) {
const char* argv[] = {"cdc_rsync.exe", kSrc, kUserHostDst, NULL};
EXPECT_TRUE(Parse(static_cast<int>(std::size(argv)) - 1, argv, &parameters_));
EXPECT_EQ(RemoteUtil::kDefaultSshPort, parameters_.options.port);
EXPECT_FALSE(parameters_.options.delete_);
EXPECT_FALSE(parameters_.options.recursive);
EXPECT_EQ(0, parameters_.options.verbosity);
EXPECT_FALSE(parameters_.options.quiet);
EXPECT_FALSE(parameters_.options.whole_file);
EXPECT_FALSE(parameters_.options.compress);
EXPECT_FALSE(parameters_.options.checksum);
EXPECT_FALSE(parameters_.options.dry_run);
EXPECT_TRUE(parameters_.options.copy_dest.empty());
EXPECT_EQ(6, parameters_.options.compress_level);
EXPECT_EQ(10, parameters_.options.connection_timeout_sec);
EXPECT_EQ(1, parameters_.sources.size());
EXPECT_EQ(parameters_.sources[0], kSrc);
EXPECT_EQ(parameters_.user_host, kUserHost);
EXPECT_EQ(parameters_.destination, kDst);
ExpectNoError();
}
TEST_F(ParamsTest, ParseSucceedsWithOptionFromTwoArguments) {
const char* argv[] = {
"cdc_rsync.exe", "--compress-level", "2", kSrc, kUserHostDst, NULL};
EXPECT_TRUE(Parse(static_cast<int>(std::size(argv)) - 1, argv, &parameters_));
EXPECT_EQ(parameters_.options.compress_level, 2);
ExpectNoError();
}
TEST_F(ParamsTest,
ParseSucceedsWithOptionFromOneArgumentWithEqualityWithValue) {
const char* argv[] = {"cdc_rsync.exe", "--compress-level=2", kSrc,
kUserHostDst, NULL};
EXPECT_TRUE(Parse(static_cast<int>(std::size(argv)) - 1, argv, &parameters_));
ASSERT_EQ(parameters_.sources.size(), 1);
EXPECT_EQ(parameters_.options.compress_level, 2);
EXPECT_EQ(parameters_.sources[0], kSrc);
EXPECT_EQ(parameters_.user_host, kUserHost);
EXPECT_EQ(parameters_.destination, kDst);
ExpectNoError();
}
TEST_F(ParamsTest, ParseFailsOnCompressLevelEqualsNoValue) {
const char* argv[] = {"cdc_rsync.exe", "--compress-level=", kSrc,
kUserHostDst, NULL};
EXPECT_FALSE(
Parse(static_cast<int>(std::size(argv)) - 1, argv, &parameters_));
ExpectError(NeedsValueError("compress-level"));
}
TEST_F(ParamsTest, ParseFailsOnPortEqualsNoValue) {
const char* argv[] = {"cdc_rsync.exe", "--port=", kSrc, kUserHostDst, NULL};
EXPECT_FALSE(
Parse(static_cast<int>(std::size(argv)) - 1, argv, &parameters_));
ExpectError(NeedsValueError("port"));
}
TEST_F(ParamsTest, ParseFailsOnContimeoutEqualsNoValue) {
const char* argv[] = {"cdc_rsync.exe", "--contimeout=", kSrc, kUserHostDst,
NULL};
EXPECT_FALSE(
Parse(static_cast<int>(std::size(argv)) - 1, argv, &parameters_));
ExpectError(NeedsValueError("contimeout"));
}
TEST_F(ParamsTest, ParseSucceedsWithSshScpCommands) {
const char* argv[] = {"cdc_rsync.exe", kSrc,
kUserHostDst, "--ssh-command=sshcmd",
"--scp-command=scpcmd", NULL};
EXPECT_TRUE(Parse(static_cast<int>(std::size(argv)) - 1, argv, &parameters_));
EXPECT_EQ(parameters_.options.scp_command, "scpcmd");
EXPECT_EQ(parameters_.options.ssh_command, "sshcmd");
}
TEST_F(ParamsTest, ParseSucceedsWithSshScpCommandsByEnvVars) {
EXPECT_OK(path::SetEnv("CDC_SSH_COMMAND", "sshcmd"));
EXPECT_OK(path::SetEnv("CDC_SCP_COMMAND", "scpcmd"));
const char* argv[] = {"cdc_rsync.exe", kSrc, kUserHostDst, NULL};
EXPECT_TRUE(Parse(static_cast<int>(std::size(argv)) - 1, argv, &parameters_));
EXPECT_EQ(parameters_.options.scp_command, "scpcmd");
EXPECT_EQ(parameters_.options.ssh_command, "sshcmd");
}
TEST_F(ParamsTest, ParseSucceedsWithNoSshCommand) {
const char* argv[] = {"cdc_rsync.exe", kSrc, kUserHostDst,
"--ssh-command=", NULL};
EXPECT_FALSE(
Parse(static_cast<int>(std::size(argv)) - 1, argv, &parameters_));
ExpectError(NeedsValueError("ssh-command"));
}
TEST_F(ParamsTest, ParseSucceedsWithNoScpCommand) {
const char* argv[] = {"cdc_rsync.exe", kSrc, kUserHostDst, "--scp-command",
NULL};
EXPECT_FALSE(
Parse(static_cast<int>(std::size(argv)) - 1, argv, &parameters_));
ExpectError(NeedsValueError("scp-command"));
}
TEST_F(ParamsTest, ParseFailsOnNoUserHost) {
const char* argv[] = {"cdc_rsync.exe", kSrc, kDst, NULL};
EXPECT_FALSE(
Parse(static_cast<int>(std::size(argv)) - 1, argv, &parameters_));
ExpectError("No remote host specified");
}
TEST_F(ParamsTest, ParseDoesNotThinkCIsAHost) {
const char* argv[] = {"cdc_rsync.exe", kSrc, "C:\\foo", NULL};
EXPECT_FALSE(
Parse(static_cast<int>(std::size(argv)) - 1, argv, &parameters_));
ExpectError("No remote host specified");
}
TEST_F(ParamsTest, ParseWithoutParametersFailsOnMissingSourceAndDestination) {
const char* argv[] = {"cdc_rsync.exe", NULL};
EXPECT_FALSE(
Parse(static_cast<int>(std::size(argv)) - 1, argv, &parameters_));
ExpectOutput("Usage:");
}
TEST_F(ParamsTest, ParseWithSingleParameterFailsOnMissingDestination) {
const char* argv[] = {"cdc_rsync.exe", kSrc, NULL};
EXPECT_FALSE(
Parse(static_cast<int>(std::size(argv)) - 1, argv, &parameters_));
ExpectError("Missing source or destination");
}
TEST_F(ParamsTest, ParseSucceedsWithMultipleLetterKeyConsumed) {
const char* argv[] = {"cdc_rsync.exe", "-rvqWRzcn", kSrc, kUserHostDst, NULL};
EXPECT_TRUE(Parse(static_cast<int>(std::size(argv)) - 1, argv, &parameters_));
EXPECT_TRUE(parameters_.options.recursive);
EXPECT_EQ(parameters_.options.verbosity, 1);
EXPECT_TRUE(parameters_.options.quiet);
EXPECT_TRUE(parameters_.options.whole_file);
EXPECT_TRUE(parameters_.options.relative);
EXPECT_TRUE(parameters_.options.compress);
EXPECT_TRUE(parameters_.options.checksum);
EXPECT_TRUE(parameters_.options.dry_run);
ExpectNoError();
}
TEST_F(ParamsTest,
ParseFailsOnMultipleLetterKeyConsumedOptionsWithUnsupportedOne) {
const char* argv[] = {"cdc_rsync.exe", "-rvqaWRzcn", kSrc, kUserHostDst,
NULL};
EXPECT_FALSE(
Parse(static_cast<int>(std::size(argv)) - 1, argv, &parameters_));
ExpectError("Unknown option: 'a'");
}
TEST_F(ParamsTest, ParseSucceedsWithMultipleLongKeyConsumedOptions) {
const char* argv[] = {"cdc_rsync.exe",
"--recursive",
"--verbosity",
"--quiet",
"--whole-file",
"--compress",
"--relative",
"--delete",
"--checksum",
"--dry-run",
"--existing",
"--json",
kSrc,
kUserHostDst,
NULL};
EXPECT_TRUE(Parse(static_cast<int>(std::size(argv)) - 1, argv, &parameters_));
EXPECT_TRUE(parameters_.options.recursive);
EXPECT_EQ(parameters_.options.verbosity, 1);
EXPECT_TRUE(parameters_.options.quiet);
EXPECT_TRUE(parameters_.options.whole_file);
EXPECT_TRUE(parameters_.options.relative);
EXPECT_TRUE(parameters_.options.compress);
EXPECT_TRUE(parameters_.options.delete_);
EXPECT_TRUE(parameters_.options.checksum);
EXPECT_TRUE(parameters_.options.dry_run);
EXPECT_TRUE(parameters_.options.existing);
EXPECT_TRUE(parameters_.options.json);
ExpectNoError();
}
TEST_F(ParamsTest, ParseFailsOnUnknownKey) {
const char* argv[] = {"cdc_rsync.exe", "-unknownKey", kSrc, kUserHostDst,
NULL};
EXPECT_FALSE(
Parse(static_cast<int>(std::size(argv)) - 1, argv, &parameters_));
ExpectError("Unknown option: 'u'");
}
TEST_F(ParamsTest, ParseSucceedsWithSupportedKeyValue) {
const char* argv[] = {
"cdc_rsync.exe", "--compress-level", "11", "--contimeout", "99", "--port",
"4086", "--copy-dest=dest", kSrc, kUserHostDst, NULL};
EXPECT_TRUE(Parse(static_cast<int>(std::size(argv)) - 1, argv, &parameters_));
EXPECT_EQ(parameters_.options.compress_level, 11);
EXPECT_EQ(parameters_.options.connection_timeout_sec, 99);
EXPECT_EQ(parameters_.options.port, 4086);
EXPECT_EQ(parameters_.options.copy_dest, "dest");
ExpectNoError();
}
TEST_F(ParamsTest, ParseSucceedsWithSupportedKeyValueWithoutEqualityForChars) {
const char* argv[] = {"cdc_rsync.exe", "--copy-dest", "dest", kSrc,
kUserHostDst, NULL};
EXPECT_TRUE(Parse(static_cast<int>(std::size(argv)) - 1, argv, &parameters_));
EXPECT_EQ(parameters_.options.copy_dest, "dest");
ExpectNoError();
}
TEST_F(ParamsTest, ParseFailsOnInvalidPort) {
const char* argv[] = {"cdc_rsync.exe", "--port=0", kSrc, kUserHostDst, NULL};
EXPECT_FALSE(
Parse(static_cast<int>(std::size(argv)) - 1, argv, &parameters_));
ExpectError("--port must specify a valid port");
}
TEST_F(ParamsTest, ParseFailsOnDeleteNeedsRecursive) {
const char* argv[] = {"cdc_rsync.exe", "--delete", kSrc, kUserHostDst, NULL};
EXPECT_FALSE(
Parse(static_cast<int>(std::size(argv)) - 1, argv, &parameters_));
ExpectError("--delete does not work without --recursive (-r)");
}
TEST_F(ParamsTest, ParseChecksCompressLevel) {
int minLevel = Options::kMinCompressLevel;
int maxLevel = Options::kMaxCompressLevel;
int levels[] = {minLevel - 1, minLevel, 0, maxLevel, maxLevel + 1};
bool valid[] = {false, true, false, true, false};
for (int n = 0; n < std::size(levels); ++n) {
std::string level = "--compress-level=" + std::to_string(levels[n]);
const char* argv[] = {"cdc_rsync.exe", level.c_str(), kSrc, kUserHostDst,
NULL};
EXPECT_EQ(Parse(static_cast<int>(std::size(argv)) - 1, argv, &parameters_),
valid[n]);
if (valid[n]) {
ExpectNoError();
} else {
ExpectError("--compress_level must be between");
}
ClearErrors();
}
}
TEST_F(ParamsTest, ParseFailsOnUnknownKeyValue) {
const char* argv[] = {"cdc_rsync.exe", "--unknownKey=5", kSrc, kUserHostDst,
NULL};
EXPECT_FALSE(
Parse(static_cast<int>(std::size(argv)) - 1, argv, &parameters_));
ExpectError("unknownKey");
}
TEST_F(ParamsTest, ParseFailsWithHelpOption) {
const char* argv[] = {"cdc_rsync.exe", kSrc, kUserHostDst, NULL};
EXPECT_TRUE(Parse(static_cast<int>(std::size(argv)) - 1, argv, &parameters_));
const char* argv2[] = {"cdc_rsync.exe", kSrc, kUserHostDst, "--help", NULL};
EXPECT_FALSE(
Parse(static_cast<int>(std::size(argv2)) - 1, argv2, &parameters_));
ExpectNoError();
const char* argv3[] = {"cdc_rsync.exe", kSrc, kUserHostDst, "-h", NULL};
EXPECT_FALSE(
Parse(static_cast<int>(std::size(argv3)) - 1, argv3, &parameters_));
ExpectNoError();
}
TEST_F(ParamsTest, ParseSucceedsWithIncludeExclude) {
const char* argv[] = {"cdc_rsync.exe",
"--include=*.txt",
"--exclude",
"*.dat",
"--include",
"*.exe",
kSrc,
kUserHostDst,
NULL};
EXPECT_TRUE(Parse(static_cast<int>(std::size(argv)) - 1, argv, &parameters_));
const std::vector<PathFilter::Rule>& rules =
parameters_.options.filter.GetRules();
ASSERT_EQ(rules.size(), 3);
ASSERT_EQ(rules[0].type, PathFilter::Rule::Type::kInclude);
ASSERT_EQ(rules[0].pattern, "*.txt");
ASSERT_EQ(rules[1].type, PathFilter::Rule::Type::kExclude);
ASSERT_EQ(rules[1].pattern, "*.dat");
ASSERT_EQ(rules[2].type, PathFilter::Rule::Type::kInclude);
ASSERT_EQ(rules[2].pattern, "*.exe");
ExpectNoError();
}
TEST_F(ParamsTest, FilesFrom_NoFile) {
const char* argv[] = {"cdc_rsync.exe", kSrc, kUserHostDst, "--files-from",
NULL};
EXPECT_FALSE(
Parse(static_cast<int>(std::size(argv)) - 1, argv, &parameters_));
ExpectError(NeedsValueError("files-from"));
}
TEST_F(ParamsTest, FilesFrom_ImpliesRelative) {
const char* argv[] = {"cdc_rsync.exe", "--files-from",
sources_file_.c_str(), base_dir_.c_str(),
kUserHostDst, NULL};
EXPECT_TRUE(Parse(static_cast<int>(std::size(argv)) - 1, argv, &parameters_));
EXPECT_TRUE(parameters_.options.relative);
ExpectNoError();
}
TEST_F(ParamsTest, FilesFrom_WithoutSourceArg) {
const char* argv[] = {"cdc_rsync.exe", "--files-from", sources_file_.c_str(),
kUserHostDst, NULL};
EXPECT_TRUE(Parse(static_cast<int>(std::size(argv)) - 1, argv, &parameters_));
EXPECT_TRUE(parameters_.options.sources_dir.empty());
EXPECT_EQ(parameters_.user_host, kUserHost);
EXPECT_EQ(parameters_.destination, kDst);
ExpectNoError();
}
TEST_F(ParamsTest, FilesFrom_WithSourceArg) {
const char* argv[] = {"cdc_rsync.exe", "--files-from",
sources_file_.c_str(), base_dir_.c_str(),
kUserHostDst, NULL};
EXPECT_TRUE(Parse(static_cast<int>(std::size(argv)) - 1, argv, &parameters_));
std::string expected_sources_dir = base_dir_;
path::EnsureEndsWithPathSeparator(&expected_sources_dir);
EXPECT_EQ(parameters_.options.sources_dir, expected_sources_dir);
EXPECT_EQ(parameters_.user_host, kUserHost);
EXPECT_EQ(parameters_.destination, kDst);
ExpectNoError();
}
TEST_F(ParamsTest, FilesFrom_ParsesFile) {
const char* argv[] = {"cdc_rsync.exe", "--files-from", sources_file_.c_str(),
kUserHostDst, NULL};
EXPECT_TRUE(Parse(static_cast<int>(std::size(argv)) - 1, argv, &parameters_));
std::vector<const char*> expected = {"file1", "file2", "file3"};
ASSERT_EQ(parameters_.sources.size(), expected.size());
for (size_t n = 0; n < expected.size(); ++n) {
EXPECT_EQ(parameters_.sources[n], expected[n]);
}
ExpectNoError();
}
TEST_F(ParamsTest, FilesFrom_EmptyFile_WithoutSourceArg) {
const char* argv[] = {"cdc_rsync.exe", "--files-from",
empty_sources_file_.c_str(), kUserHostDst, NULL};
EXPECT_FALSE(
Parse(static_cast<int>(std::size(argv)) - 1, argv, &parameters_));
ExpectError(empty_sources_file_);
ExpectError("--files-from option is empty");
}
TEST_F(ParamsTest, FilesFrom_EmptyFile_WithSourceArg) {
const char* argv[] = {
"cdc_rsync.exe", "--files-from", empty_sources_file_.c_str(),
base_dir_.c_str(), kUserHostDst, NULL};
EXPECT_FALSE(
Parse(static_cast<int>(std::size(argv)) - 1, argv, &parameters_));
ExpectError(empty_sources_file_);
ExpectError("--files-from option is empty");
}
TEST_F(ParamsTest, FilesFrom_NoDestination) {
const char* argv[] = {"cdc_rsync.exe", "--files-from", sources_file_.c_str(),
NULL};
EXPECT_FALSE(
Parse(static_cast<int>(std::size(argv)) - 1, argv, &parameters_));
ExpectError("Missing destination");
}
TEST_F(ParamsTest, IncludeFrom_NoFile) {
const char* argv[] = {"cdc_rsync.exe", kSrc, kUserHostDst, "--include-from",
NULL};
EXPECT_FALSE(
Parse(static_cast<int>(std::size(argv)) - 1, argv, &parameters_));
ExpectError(NeedsValueError("include-from"));
}
TEST_F(ParamsTest, IncludeFrom_ParsesFile) {
std::string file = path::Join(base_dir_, "include_files.txt");
const char* argv[] = {"cdc_rsync.exe", "--include-from",
file.c_str(), kSrc,
kUserHostDst, NULL};
EXPECT_TRUE(Parse(static_cast<int>(std::size(argv)) - 1, argv, &parameters_));
const std::vector<PathFilter::Rule>& rules =
parameters_.options.filter.GetRules();
ASSERT_EQ(rules.size(), 1);
ASSERT_EQ(rules[0].type, PathFilter::Rule::Type::kInclude);
ASSERT_EQ(rules[0].pattern, "file3");
ExpectNoError();
}
TEST_F(ParamsTest, ExcludeFrom_NoFile) {
const char* argv[] = {"cdc_rsync.exe", kSrc, kUserHostDst, "--exclude-from",
NULL};
EXPECT_FALSE(
Parse(static_cast<int>(std::size(argv)) - 1, argv, &parameters_));
ExpectError(NeedsValueError("exclude-from"));
}
TEST_F(ParamsTest, ExcludeFrom_ParsesFile) {
std::string file = path::Join(base_dir_, "exclude_files.txt");
const char* argv[] = {"cdc_rsync.exe", "--exclude-from",
file.c_str(), kSrc,
kUserHostDst, NULL};
EXPECT_TRUE(Parse(static_cast<int>(std::size(argv)) - 1, argv, &parameters_));
const std::vector<PathFilter::Rule>& rules =
parameters_.options.filter.GetRules();
ASSERT_EQ(rules.size(), 2);
EXPECT_EQ(rules[0].type, PathFilter::Rule::Type::kExclude);
EXPECT_EQ(rules[0].pattern, "file1");
EXPECT_EQ(rules[1].type, PathFilter::Rule::Type::kExclude);
EXPECT_EQ(rules[1].pattern, "file2");
ExpectNoError();
}
TEST_F(ParamsTest, IncludeExcludeMixed_ProperOrder) {
std::string exclude_file = path::Join(base_dir_, "exclude_files.txt");
std::string include_file = path::Join(base_dir_, "include_files.txt");
const char* argv[] = {"cdc_rsync.exe",
"--include-from",
include_file.c_str(),
"--exclude=excl1",
kSrc,
"--exclude-from",
exclude_file.c_str(),
kUserHostDst,
"--include",
"incl1",
NULL};
EXPECT_TRUE(Parse(static_cast<int>(std::size(argv)) - 1, argv, &parameters_));
const std::vector<PathFilter::Rule>& rules =
parameters_.options.filter.GetRules();
ASSERT_EQ(rules.size(), 5);
EXPECT_EQ(rules[0].type, PathFilter::Rule::Type::kInclude);
EXPECT_EQ(rules[0].pattern, "file3");
EXPECT_EQ(rules[1].type, PathFilter::Rule::Type::kExclude);
EXPECT_EQ(rules[1].pattern, "excl1");
EXPECT_EQ(rules[2].type, PathFilter::Rule::Type::kExclude);
EXPECT_EQ(rules[2].pattern, "file1");
EXPECT_EQ(rules[3].type, PathFilter::Rule::Type::kExclude);
EXPECT_EQ(rules[3].pattern, "file2");
EXPECT_EQ(rules[4].type, PathFilter::Rule::Type::kInclude);
EXPECT_EQ(rules[4].pattern, "incl1");
ExpectNoError();
}
} // namespace
} // namespace params
} // namespace cdc_ft

3
cdc_rsync/testdata/params/empty_source_files.txt vendored Normal file
View File

@@ -0,0 +1,3 @@

2
cdc_rsync/testdata/params/exclude_files.txt vendored Normal file
View File

@@ -0,0 +1,2 @@
file1
file2

1
cdc_rsync/testdata/params/include_files.txt vendored Normal file
View File

@@ -0,0 +1 @@
file3

6
cdc_rsync/testdata/params/source_files.txt vendored Normal file
View File

@@ -0,0 +1,6 @@
file1
file2
file3