github-mirrors/netris-cdc-file-transfer
1
0
Fork 0
mirror of https://github.com/nestriness/cdc-file-transfer.git synced 2026-01-30 08:55:36 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge cdc_stream into asset_stream_manager (#26)

Browse Source 
Adds start and stop commands to asset_stream_manager.
asset_stream_manager will be renamed to cdc_stream next.
This commit is contained in:
Lutz Justen
2022-12-01 12:40:00 +01:00
committed by GitHub
parent 876e59409f
commit f0539226a2
23 changed files with 507 additions and 281 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
.github/workflows/build_and_test.yml vendored
View File

@@ -45,7 +45,7 @@ jobs:
- name: Build
run: |
bazel build --config=windows //cdc_rsync //cdc_stream //asset_stream_manager //tests_common //tests_asset_streaming_30 //tests_cdc_rsync
bazel build --config=windows //cdc_rsync //asset_stream_manager //tests_common //tests_asset_streaming_30 //tests_cdc_rsync
- name: Test
run: |

3
.github/workflows/create_release.yml vendored
View File

@@ -53,7 +53,7 @@ jobs:
- name: Build
run: |
bazel build --config=windows --compilation_mode=opt --copt=/GL `
//cdc_rsync //cdc_stream //asset_stream_manager //tests_common //tests_asset_streaming_30 //tests_cdc_rsync
//cdc_rsync //asset_stream_manager //tests_common //tests_asset_streaming_30 //tests_cdc_rsync
- name: Test
run: |
@@ -77,7 +77,6 @@ jobs:
run: |
mkdir artifacts
cp bazel-bin/cdc_rsync/cdc_rsync.exe artifacts
cp bazel-bin/cdc_stream/cdc_stream.exe artifacts
cp bazel-bin/asset_stream_manager/asset_stream_manager.exe artifacts
cp LICENSE artifacts
cp README.md artifacts

10
all_files.vcxitems
View File

@@ -21,6 +21,7 @@
<ClCompile Include="$(MSBuildThisFileDirectory)asset_stream_manager\base_command.cc" />
<ClCompile Include="$(MSBuildThisFileDirectory)asset_stream_manager\cdc_fuse_manager.cc" />
<ClCompile Include="$(MSBuildThisFileDirectory)asset_stream_manager\grpc_asset_stream_server.cc" />
<ClCompile Include="$(MSBuildThisFileDirectory)asset_stream_manager\local_assets_stream_manager_client.cc" />
<ClCompile Include="$(MSBuildThisFileDirectory)asset_stream_manager\local_assets_stream_manager_service_impl.cc" />
<ClCompile Include="$(MSBuildThisFileDirectory)asset_stream_manager\main.cc" />
<ClCompile Include="$(MSBuildThisFileDirectory)asset_stream_manager\metrics_recorder.cc" />
@@ -30,7 +31,9 @@
<ClCompile Include="$(MSBuildThisFileDirectory)asset_stream_manager\session.cc" />
<ClCompile Include="$(MSBuildThisFileDirectory)asset_stream_manager\session_management_server.cc" />
<ClCompile Include="$(MSBuildThisFileDirectory)asset_stream_manager\session_manager.cc" />
<ClCompile Include="$(MSBuildThisFileDirectory)asset_stream_manager\start_command.cc" />
<ClCompile Include="$(MSBuildThisFileDirectory)asset_stream_manager\start_service_command.cc" />
<ClInclude Include="$(MSBuildThisFileDirectory)asset_stream_manager\stop_command.cc" />
<ClCompile Include="$(MSBuildThisFileDirectory)asset_stream_manager\testing_asset_stream_server.cc" />
<ClCompile Include="$(MSBuildThisFileDirectory)cdc_fuse_fs\asset.cc" />
<ClCompile Include="$(MSBuildThisFileDirectory)cdc_fuse_fs\asset_stream_client.cc" />
@@ -42,8 +45,6 @@
<ClCompile Include="$(MSBuildThisFileDirectory)cdc_fuse_fs\mock_libfuse.cc" />
<ClCompile Include="$(MSBuildThisFileDirectory)cdc_indexer\indexer.cc" />
<ClCompile Include="$(MSBuildThisFileDirectory)cdc_indexer\main.cc" />
<ClCompile Include="$(MSBuildThisFileDirectory)cdc_stream\local_assets_stream_manager_client.cc" />
<ClCompile Include="$(MSBuildThisFileDirectory)cdc_stream\main.cc" />
<ClCompile Include="$(MSBuildThisFileDirectory)common\buffer.cc" />
<ClCompile Include="$(MSBuildThisFileDirectory)common\buffer_test.cc" />
<ClCompile Include="$(MSBuildThisFileDirectory)common\clock.cc" />
@@ -150,6 +151,7 @@
<ClInclude Include="$(MSBuildThisFileDirectory)asset_stream_manager\base_command.h" />
<ClInclude Include="$(MSBuildThisFileDirectory)asset_stream_manager\cdc_fuse_manager.h" />
<ClInclude Include="$(MSBuildThisFileDirectory)asset_stream_manager\grpc_asset_stream_server.h" />
<ClInclude Include="$(MSBuildThisFileDirectory)asset_stream_manager\local_assets_stream_manager_client.h" />
<ClInclude Include="$(MSBuildThisFileDirectory)asset_stream_manager\local_assets_stream_manager_service_impl.h" />
<ClInclude Include="$(MSBuildThisFileDirectory)asset_stream_manager\metrics_recorder.h" />
<ClInclude Include="$(MSBuildThisFileDirectory)asset_stream_manager\multi_session.h" />
@@ -157,7 +159,9 @@
<ClInclude Include="$(MSBuildThisFileDirectory)asset_stream_manager\session_config.h" />
<ClInclude Include="$(MSBuildThisFileDirectory)asset_stream_manager\session_management_server.h" />
<ClInclude Include="$(MSBuildThisFileDirectory)asset_stream_manager\session_manager.h" />
<ClInclude Include="$(MSBuildThisFileDirectory)asset_stream_manager\start_command.h" />
<ClInclude Include="$(MSBuildThisFileDirectory)asset_stream_manager\start_service_command.h" />
<ClInclude Include="$(MSBuildThisFileDirectory)asset_stream_manager\stop_command.h" />
<ClInclude Include="$(MSBuildThisFileDirectory)asset_stream_manager\testing_asset_stream_server.h" />
<ClInclude Include="$(MSBuildThisFileDirectory)cdc_fuse_fs\asset.h" />
<ClInclude Include="$(MSBuildThisFileDirectory)cdc_fuse_fs\asset_stream_client.h" />
@@ -166,7 +170,6 @@
<ClInclude Include="$(MSBuildThisFileDirectory)cdc_fuse_fs\constants.h" />
<ClInclude Include="$(MSBuildThisFileDirectory)cdc_fuse_fs\mock_libfuse.h" />
<ClInclude Include="$(MSBuildThisFileDirectory)cdc_indexer\indexer.h" />
<ClInclude Include="$(MSBuildThisFileDirectory)cdc_stream\local_assets_stream_manager_client.h" />
<ClInclude Include="$(MSBuildThisFileDirectory)common\buffer.h" />
<ClInclude Include="$(MSBuildThisFileDirectory)common\clock.h" />
<ClInclude Include="$(MSBuildThisFileDirectory)common\dir_iter.h" />
@@ -246,7 +249,6 @@
<None Include="$(MSBuildThisFileDirectory)cdc_indexer\BUILD" />
<None Include="$(MSBuildThisFileDirectory)cdc_indexer\README.md" />
<None Include="$(MSBuildThisFileDirectory)cdc_rsync\protos\BUILD" />
<None Include="$(MSBuildThisFileDirectory)cdc_stream\BUILD" />
<None Include="$(MSBuildThisFileDirectory)common\BUILD" />
<None Include="$(MSBuildThisFileDirectory)data_store\BUILD" />
<None Include="$(MSBuildThisFileDirectory)cdc_rsync\base\BUILD" />

68
asset_stream_manager/BUILD
View File

@@ -9,27 +9,71 @@ cc_binary(
srcs = ["main.cc"],
data = [":roots_pem"],
deps = [
":commands",
"//cdc_stream",
":start_command",
":start_service_command",
":stop_command",
"//common:log",
"//common:path",
],
)
cc_library(
name = "commands",
srcs = [
"base_command.cc",
"start_service_command.cc",
],
hdrs = [
"base_command.h",
"start_service_command.h",
name = "base_command",
srcs = ["base_command.cc"],
hdrs = ["base_command.h"],
deps = [
"//absl_helper:jedec_size_flag",
"@com_github_lyra//:lyra",
"@com_google_absl//absl/status",
"@com_google_absl//absl/strings:str_format",
],
)
cc_library(
name = "start_service_command",
srcs = ["start_service_command.cc"],
hdrs = ["start_service_command.h"],
deps = [
":asset_stream_config",
":base_command",
":session_management_server",
"@com_github_lyra//:lyra",
],
)
cc_library(
name = "start_command",
srcs = ["start_command.cc"],
hdrs = ["start_command.h"],
deps = [
":base_command",
":local_assets_stream_manager_client",
":session_management_server",
"//common:path",
"//common:status_macros",
],
)
cc_library(
name = "stop_command",
srcs = ["stop_command.cc"],
hdrs = ["stop_command.h"],
deps = [
":base_command",
":local_assets_stream_manager_client",
":session_management_server",
"//common:path",
"//common:remote_util",
"//common:status_macros",
],
)
cc_library(
name = "local_assets_stream_manager_client",
srcs = ["local_assets_stream_manager_client.cc"],
hdrs = ["local_assets_stream_manager_client.h"],
deps = [
"//common:grpc_status",
"//proto:local_assets_stream_manager_grpc_proto",
"@com_google_absl//absl/status",
],
)
@@ -66,8 +110,8 @@ cc_library(
srcs = ["asset_stream_config.cc"],
hdrs = ["asset_stream_config.h"],
deps = [
":base_command",
":multi_session",
"//absl_helper:jedec_size_flag",
"//common:log",
"//common:path",
"//common:status_macros",

33
asset_stream_manager/asset_stream_config.cc
View File

@@ -19,6 +19,7 @@
#include "absl/strings/str_format.h"
#include "absl/strings/str_join.h"
#include "absl_helper/jedec_size_flag.h"
#include "asset_stream_manager/base_command.h"
#include "common/buffer.h"
#include "common/path.h"
#include "common/status_macros.h"
@@ -38,12 +39,14 @@ AssetStreamConfig::AssetStreamConfig() = default;
AssetStreamConfig::~AssetStreamConfig() = default;
void AssetStreamConfig::RegisterCommandLineFlags(lyra::command& cmd) {
void AssetStreamConfig::RegisterCommandLineFlags(lyra::command& cmd,
BaseCommand& base_command) {
session_cfg_.verbosity = kDefaultVerbosity;
cmd.add_argument(lyra::opt(session_cfg_.verbosity, "num")
.name("--verbosity")
.help("Verbosity of the log output, default: " +
std::to_string(kDefaultVerbosity)));
std::to_string(kDefaultVerbosity) +
". Increase to make logs more verbose."));
cmd.add_argument(
lyra::opt(session_cfg_.stats)
@@ -84,13 +87,14 @@ void AssetStreamConfig::RegisterCommandLineFlags(lyra::command& cmd) {
.help("Check FUSE consistency and log check results"));
session_cfg_.fuse_cache_capacity = DiskDataStore::kDefaultCapacity;
cmd.add_argument(lyra::opt(JedecParser("--cache-capacity",
cmd.add_argument(
lyra::opt(base_command.JedecParser("--cache-capacity",
&session_cfg_.fuse_cache_capacity),
"bytes")
.name("--cache-capacity")
.help("FUSE cache capacity, default: " +
std::to_string(DiskDataStore::kDefaultCapacity) +
". Supports common unit suffixes K, M, G."));
"bytes")
.name("--cache-capacity")
.help("FUSE cache capacity, default: " +
std::to_string(DiskDataStore::kDefaultCapacity) +
". Supports common unit suffixes K, M, G."));
session_cfg_.fuse_cleanup_timeout_sec = DataProvider::kCleanupTimeoutSec;
cmd.add_argument(
@@ -251,18 +255,5 @@ std::string AssetStreamConfig::GetFlagReadErrors() {
error_str.empty() ? "" : "\n", flag, error);
return error_str;
}
std::function<void(const std::string&)> AssetStreamConfig::JedecParser(
const char* flag_name, uint64_t* bytes) {
return [flag_name, bytes,
error = &jedec_parse_error_](const std::string& value) {
JedecSize size;
if (AbslParseFlag(value, &size, error)) {
*bytes = size.Size();
} else {
*error = absl::StrFormat("Failed to parse --%s=%s: %s", flag_name, value,
*error);
}
};
}
} // namespace cdc_ft

16
asset_stream_manager/asset_stream_config.h
View File

@@ -31,6 +31,8 @@ class command;
namespace cdc_ft {
class BaseCommand;
// Class containing all configuration settings for asset streaming.
// Reads flags from the command line and optionally applies overrides from
// a json file.
@@ -41,7 +43,7 @@ class AssetStreamConfig {
~AssetStreamConfig();
// Registers arguments with Lyra.
void RegisterCommandLineFlags(lyra::command& cmd);
void RegisterCommandLineFlags(lyra::command& cmd, BaseCommand& base_command);
// Loads a configuration from the JSON file at |path| and overrides any config
// values that are set in this file. Sample json file:
@@ -88,16 +90,7 @@ class AssetStreamConfig {
// Whether to log to a file or to stdout.
bool log_to_stdout() const { return log_to_stdout_; }
// Workaround for Lyra not accepting errors from parsers.
const std::string& jedec_parse_error() const { return jedec_parse_error_; }
private:
// Jedec parser for Lyra options. Usage:
// lyra::opt(JedecParser("size-flag", &size_bytes), "bytes"))
// Sets jedec_parse_error_ on error, Lyra doesn't support errors from lambdas.
std::function<void(const std::string&)> JedecParser(const char* flag_name,
uint64_t* bytes);
SessionConfig session_cfg_;
bool log_to_stdout_ = false;
@@ -111,9 +104,6 @@ class AssetStreamConfig {
// Maps flags to errors occurred while reading this flag.
std::map<std::string, std::string> flag_read_errors_;
// Errors from parsing JEDEC sizes.
std::string jedec_parse_error_;
};
}; // namespace cdc_ft

2
asset_stream_manager/asset_stream_manager.vcxproj
View File

@@ -66,7 +66,7 @@
</ItemDefinitionGroup>
<!-- Bazel setup -->
<PropertyGroup>
<BazelTargets>//asset_stream_manager //cdc_stream</BazelTargets>
<BazelTargets>//asset_stream_manager</BazelTargets>
<BazelOutputFile>asset_stream_manager.exe</BazelOutputFile>
<BazelIncludePaths>..\;..\third_party\absl;..\bazel-cdc-file-transfer\external\com_github_jsoncpp\include;..\bazel-cdc-file-transfer\external\com_github_blake3\c;..\third_party\googletest\googletest\include;..\bazel-cdc-file-transfer\external\com_google_protobuf\src;..\bazel-cdc-file-transfer\external\com_github_grpc_grpc\include;..\bazel-out\x64_windows-dbg\bin;..\bazel-cdc-file-transfer\external\com_github_lyra\include;$(VC_IncludePath);$(WindowsSDK_IncludePath)</BazelIncludePaths>
</PropertyGroup>

45
asset_stream_manager/base_command.cc
View File

@@ -14,6 +14,8 @@
#include "asset_stream_manager/base_command.h"
#include "absl/strings/str_format.h"
#include "absl_helper/jedec_size_flag.h"
#include "lyra/lyra.hpp"
namespace cdc_ft {
@@ -33,14 +35,38 @@ void BaseCommand::Register(lyra::cli& cli) {
RegisterCommandLineFlags(cmd);
// Workaround for Lyra treating --unknown_flags as positional argument.
// If this argument is not empty, it's an unknown arg.
cmd.add_argument(lyra::arg(invalid_arg_, ""));
// Detect extra positional args.
cmd.add_argument(lyra::arg(PosArgValidator(&extra_positional_arg_), ""));
// Register command with CLI.
cli.add_argument(std::move(cmd));
}
std::function<void(const std::string&)> BaseCommand::JedecParser(
const char* flag_name, uint64_t* bytes) {
return [flag_name, bytes,
error = &jedec_parse_error_](const std::string& value) {
JedecSize size;
if (AbslParseFlag(value, &size, error)) {
*bytes = size.Size();
} else {
*error = absl::StrFormat("Failed to parse %s=%s: %s", flag_name, value,
*error);
}
};
}
std::function<void(const std::string&)> BaseCommand::PosArgValidator(
std::string* str) {
return [str, invalid_arg = &invalid_arg_](const std::string& value) {
if (!value.empty() && value[0] == '-') {
*invalid_arg = value;
} else {
*str = value;
}
};
}
void BaseCommand::CommandHandler(const lyra::group& g) {
// Handle -h, --help.
if (show_help_) {
@@ -57,6 +83,19 @@ void BaseCommand::CommandHandler(const lyra::group& g) {
return;
}
if (!jedec_parse_error_.empty()) {
std::cerr << "Error: " << jedec_parse_error_ << std::endl;
*exit_code_ = 1;
return;
}
if (!extra_positional_arg_.empty()) {
std::cerr << "Error: Extraneous positional argument '"
<< extra_positional_arg_ << "'. Try -h for help." << std::endl;
*exit_code_ = 1;
return;
}
// Run and print error.
absl::Status status = Run();
if (!status.ok()) {

19
asset_stream_manager/base_command.h
View File

@@ -42,6 +42,18 @@ class BaseCommand {
// Registers the command with Lyra. Must be called before parsing args.
void Register(lyra::cli& cli);
// Jedec parser for Lyra options. Usage:
// lyra::opt(JedecParser("size-flag", &size_bytes), "bytes"))
// Automatically reports a parse failure on error.
std::function<void(const std::string&)> JedecParser(const char* flag_name,
uint64_t* bytes);
// Validator that should be used for all positional arguments. Lyra interprets
// -u, --unknown_flag as positional argument. This validator makes sure that
// a positional argument starting with - is reported as an error. Otherwise,
// writes the value to |str|.
std::function<void(const std::string&)> PosArgValidator(std::string* str);
protected:
// Adds all optional and required arguments used by the command.
// Called by Register().
@@ -66,6 +78,13 @@ class BaseCommand {
// Workaround for invalid args. Lyra doesn't interpret --invalid as invalid
// argument, but as positional argument "--invalid".
std::string invalid_arg_;
// Extraneous positional args. Gets reported as error if present.
std::string extra_positional_arg_;
// Errors from parsing JEDEC sizes.
// Works around Lyra not accepting errors from parsers.
std::string jedec_parse_error_;
};
} // namespace cdc_ft

35
cdc_stream/local_assets_stream_manager_client.cc → asset_stream_manager/local_assets_stream_manager_client.cc
View File

@@ -12,9 +12,13 @@
// See the License for the specific language governing permissions and
// limitations under the License.
#include "cdc_stream/local_assets_stream_manager_client.h"
#include "asset_stream_manager/local_assets_stream_manager_client.h"
#include <vector>
#include "absl/status/status.h"
#include "absl/strings/str_format.h"
#include "absl/strings/str_split.h"
#include "common/grpc_status.h"
namespace cdc_ft {
@@ -24,6 +28,15 @@ using StartSessionResponse = localassetsstreammanager::StartSessionResponse;
using StopSessionRequest = localassetsstreammanager::StopSessionRequest;
using StopSessionResponse = localassetsstreammanager::StopSessionResponse;
LocalAssetsStreamManagerClient::LocalAssetsStreamManagerClient(
uint16_t service_port) {
std::string client_address = absl::StrFormat("localhost:%u", service_port);
std::shared_ptr<grpc::Channel> channel = grpc::CreateCustomChannel(
client_address, grpc::InsecureChannelCredentials(),
grpc::ChannelArguments());
stub_ = LocalAssetsStreamManager::NewStub(std::move(channel));
}
LocalAssetsStreamManagerClient::LocalAssetsStreamManagerClient(
std::shared_ptr<grpc::Channel> channel) {
stub_ = LocalAssetsStreamManager::NewStub(std::move(channel));
@@ -59,4 +72,24 @@ absl::Status LocalAssetsStreamManagerClient::StopSession(
return ToAbslStatus(stub_->StopSession(&context, request, &response));
}
// static
absl::Status LocalAssetsStreamManagerClient::ParseUserHostDir(
const std::string& user_host_dir, std::string* user_host,
std::string* dir) {
std::vector<std::string> parts =
absl::StrSplit(user_host_dir, absl::MaxSplits(':', 1));
if (parts.size() < 2 ||
(parts[0].size() == 1 && toupper(parts[0][0]) >= 'A' &&
toupper(parts[0][0]) <= 'Z')) {
return absl::InvalidArgumentError(
absl::StrFormat("Failed to parse '%s'. Make sure it is of the form "
"[user@]host:linux_dir.",
user_host_dir));
}
*user_host = parts[0];
*dir = parts[1];
return absl::OkStatus();
}
} // namespace cdc_ft

26
cdc_stream/local_assets_stream_manager_client.h → asset_stream_manager/local_assets_stream_manager_client.h
View File

@@ -32,26 +32,40 @@ namespace cdc_ft {
// gRpc client for starting/stopping asset streaming sessions.
class LocalAssetsStreamManagerClient {
public:
explicit LocalAssetsStreamManagerClient(uint16_t service_port);
// |channel| is a grpc channel to use.
explicit LocalAssetsStreamManagerClient(
std::shared_ptr<grpc::Channel> channel);
~LocalAssetsStreamManagerClient();
// Starts streaming the Windows directory |src_dir| to the Linux target
// |user_host_dir|, which must be formatted as [user@]host:dir, e.g.
// jdoe@jdoe.corp.foo.com:~/assets
// Starts streaming |src_dir| to |user_host|:|mount_dir|.
// Starting a second session to the same target will stop the first one.
// |src_dir| is the Windows source directory to stream.
// |user_host| is the Linux host, formatted as [user@:host].
// |ssh_port| is the SSH port to use while connecting to the host.
// |mount_dir| is the Linux target directory to stream to.
// |ssh_command| is the ssh command and extra arguments to use.
// |scp_command| is the scp command and extra arguments to use.
absl::Status StartSession(const std::string& src_dir,
const std::string& user_host, uint16_t ssh_port,
const std::string& mount_dir,
const std::string& ssh_command,
const std::string& scp_command);
// Stops the streaming session to the Linux target |user_host_dir|, which must
// be formatted as [user@]host:dir, e.g. jdoe@jdoe.corp.foo.com:~/assets
absl::Status StopSession(const std::string& user_host_dir,
// Stops the streaming session to the Linux target |user_host|:|mount_dir|.
// |user_host| is the Linux host, formatted as [user@:host].
// |mount_dir| is the Linux target directory.
absl::Status StopSession(const std::string& user_host,
const std::string& mount_dir);
// Helper function that splits "user@host:dir" into "user@host" and "dir".
// Does not think that C: is a host.
static absl::Status ParseUserHostDir(const std::string& user_host_dir,
std::string* user_host,
std::string* dir);
private:
using LocalAssetsStreamManager =
localassetsstreammanager::LocalAssetsStreamManager;

12
asset_stream_manager/main.cc
View File

@@ -12,7 +12,9 @@
// See the License for the specific language governing permissions and
// limitations under the License.
#include "asset_stream_manager/start_command.h"
#include "asset_stream_manager/start_service_command.h"
#include "asset_stream_manager/stop_command.h"
#include "lyra/lyra.hpp"
int main(int argc, char* argv[]) {
@@ -22,8 +24,14 @@ int main(int argc, char* argv[]) {
int exit_code = -1;
cli.add_argument(lyra::help(show_help));
cdc_ft::StartServiceCommand start_service(&exit_code);
start_service.Register(cli);
cdc_ft::StartCommand start_cmd(&exit_code);
start_cmd.Register(cli);
cdc_ft::StopCommand stop_cmd(&exit_code);
stop_cmd.Register(cli);
cdc_ft::StartServiceCommand start_service_cmd(&exit_code);
start_service_cmd.Register(cli);
// Parse args and run. Note that parse actually runs the commands.
// exit_code is -1 if no command was run.

2
asset_stream_manager/session_management_server.h
View File

@@ -36,6 +36,8 @@ class ProcessFactory;
// - Background
class SessionManagementServer {
public:
static constexpr int kDefaultServicePort = 44432;
SessionManagementServer(grpc::Service* session_service,
grpc::Service* background_service,
SessionManager* session_manager);

110
asset_stream_manager/start_command.cc Normal file
View File

@@ -0,0 +1,110 @@
// Copyright 2022 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "asset_stream_manager/start_command.h"
#include <memory>
#include "asset_stream_manager/local_assets_stream_manager_client.h"
#include "asset_stream_manager/session_management_server.h"
#include "common/log.h"
#include "common/path.h"
#include "common/remote_util.h"
#include "common/status_macros.h"
#include "lyra/lyra.hpp"
namespace cdc_ft {
namespace {
constexpr int kDefaultVerbosity = 2;
} // namespace
StartCommand::StartCommand(int* exit_code)
: BaseCommand("start",
"Start streaming files from a Windows to a Linux device",
exit_code) {}
StartCommand::~StartCommand() = default;
void StartCommand::RegisterCommandLineFlags(lyra::command& cmd) {
verbosity_ = kDefaultVerbosity;
cmd.add_argument(lyra::opt(verbosity_, "num")
.name("--verbosity")
.help("Verbosity of the log output, default: " +
std::to_string(kDefaultVerbosity) +
". Increase to make logs more verbose."));
service_port_ = SessionManagementServer::kDefaultServicePort;
cmd.add_argument(
lyra::opt(service_port_, "port")
.name("--service-port")
.help("Local port to use while connecting to the local "
"asset stream service, default: " +
std::to_string(SessionManagementServer::kDefaultServicePort)));
ssh_port_ = RemoteUtil::kDefaultSshPort;
cmd.add_argument(
lyra::opt(ssh_port_, "port")
.name("--ssh-port")
.help("Port to use while connecting to the remote instance being "
"streamed to, default: " +
std::to_string(RemoteUtil::kDefaultSshPort)));
path::GetEnv("CDC_SSH_COMMAND", &ssh_command_).IgnoreError();
cmd.add_argument(
lyra::opt(ssh_command_, "ssh_command")
.name("--ssh-command")
.help("Path and arguments of ssh command to use, e.g. "
"\"C:\\path\\to\\ssh.exe -F config_file\". Can also be "
"specified by the CDC_SSH_COMMAND environment variable."));
path::GetEnv("CDC_SCP_COMMAND", &scp_command_).IgnoreError();
cmd.add_argument(
lyra::opt(scp_command_, "scp_command")
.name("--scp-command")
.help("Path and arguments of scp command to use, e.g. "
"\"C:\\path\\to\\scp.exe -F config_file\". Can also be "
"specified by the CDC_SCP_COMMAND environment variable."));
cmd.add_argument(lyra::arg(PosArgValidator(&src_dir_), "dir")
.required()
.help("Windows directory to stream"));
cmd.add_argument(
lyra::arg(PosArgValidator(&user_host_dir_), "[user@]host:src-dir")
.required()
.help("Linux host and directory to stream to"));
}
absl::Status StartCommand::Run() {
LogLevel level = Log::VerbosityToLogLevel(verbosity_);
ScopedLog scoped_log(std::make_unique<ConsoleLog>(level));
LocalAssetsStreamManagerClient client(service_port_);
std::string full_src_dir = path::GetFullPath(src_dir_);
std::string user_host, mount_dir;
RETURN_IF_ERROR(LocalAssetsStreamManagerClient::ParseUserHostDir(
user_host_dir_, &user_host, &mount_dir));
absl::Status status =
client.StartSession(full_src_dir, user_host, ssh_port_, mount_dir,
ssh_command_, scp_command_);
if (status.ok()) {
LOG_INFO("Started streaming directory '%s' to '%s:%s'", src_dir_, user_host,
mount_dir);
}
return status;
}
} // namespace cdc_ft

48
asset_stream_manager/start_command.h Normal file
View File

@@ -0,0 +1,48 @@
/*
* Copyright 2022 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef ASSET_STREAM_MANAGER_START_COMMAND_H_
#define ASSET_STREAM_MANAGER_START_COMMAND_H_
#include "absl/status/status.h"
#include "asset_stream_manager/base_command.h"
namespace cdc_ft {
// Handler for the start command. Sends an RPC call to the service to starts a
// new asset streaming session.
class StartCommand : public BaseCommand {
public:
explicit StartCommand(int* exit_code);
~StartCommand();
// BaseCommand:
void RegisterCommandLineFlags(lyra::command& cmd) override;
absl::Status Run() override;
private:
int verbosity_ = 0;
uint16_t service_port_ = 0;
uint16_t ssh_port_ = 0;
std::string ssh_command_;
std::string scp_command_;
std::string src_dir_;
std::string user_host_dir_;
};
} // namespace cdc_ft
#endif // ASSET_STREAM_MANAGER_START_COMMAND_H_

26
asset_stream_manager/start_service_command.cc
View File

@@ -30,8 +30,6 @@
namespace cdc_ft {
namespace {
constexpr int kSessionManagementPort = 44432;
std::string GetLogPath(const char* log_dir, const char* log_base_name) {
DefaultSystemClock* clock = DefaultSystemClock::GetInstance();
std::string timestamp_ext = clock->FormatNow(".%Y%m%d-%H%M%S.log", false);
@@ -57,19 +55,17 @@ void StartServiceCommand::RegisterCommandLineFlags(lyra::command& cmd) {
.name("--log-dir")
.help("Directory to store log files, default: " + log_dir_));
cfg_.RegisterCommandLineFlags(cmd);
cfg_.RegisterCommandLineFlags(cmd, *this);
}
absl::Status StartServiceCommand::Run() {
if (!cfg_.jedec_parse_error().empty()) {
return absl::InvalidArgumentError(cfg_.jedec_parse_error());
}
// Set up config. Allow overriding this config with |config_file|.
absl::Status cfg_load_status = path::ExpandPathVariables(&config_file_);
cfg_load_status.Update(cfg_.LoadFromFile(config_file_));
RETURN_IF_ERROR(InitLogging());
std::unique_ptr<Log> logger;
ASSIGN_OR_RETURN(logger, GetLogger());
cdc_ft::ScopedLog scoped_log(std::move(logger));
// Log status of loaded configuration. Errors are not critical.
if (cfg_load_status.ok()) {
@@ -102,16 +98,14 @@ absl::Status StartServiceCommand::Run() {
LOG_INFO("Asset stream manager shut down successfully.");
}
Log::Shutdown();
return status;
}
absl::Status StartServiceCommand::InitLogging() {
absl::StatusOr<std::unique_ptr<Log>> StartServiceCommand::GetLogger() {
LogLevel level = Log::VerbosityToLogLevel(cfg_.session_cfg().verbosity);
if (cfg_.log_to_stdout()) {
// Log to stdout.
Log::Initialize(std::make_unique<ConsoleLog>(level));
return absl::OkStatus();
return std::make_unique<ConsoleLog>(level);
}
// Log to file.
@@ -121,9 +115,8 @@ absl::Status StartServiceCommand::InitLogging() {
absl::StrFormat("Failed to create log directory '%s'", log_dir_));
}
Log::Initialize(std::make_unique<FileLog>(
level, GetLogPath(log_dir_.c_str(), "assets_stream_manager").c_str()));
return absl::OkStatus();
return std::make_unique<FileLog>(
level, GetLogPath(log_dir_.c_str(), "assets_stream_manager").c_str());
}
// Runs the session management service and returns when it finishes.
@@ -154,7 +147,8 @@ absl::Status StartServiceCommand::RunService() {
RETURN_ABSL_IF_ERROR(
session_service.StartSession(nullptr, &request, &response));
}
RETURN_IF_ERROR(sm_server.Start(kSessionManagementPort));
RETURN_IF_ERROR(
sm_server.Start(SessionManagementServer::kDefaultServicePort));
sm_server.RunUntilShutdown();
return absl::OkStatus();
}

9
asset_stream_manager/start_service_command.h
View File

@@ -17,6 +17,10 @@
#ifndef ASSET_STREAM_MANAGER_START_SERVICE_COMMAND_H_
#define ASSET_STREAM_MANAGER_START_SERVICE_COMMAND_H_
#include <memory>
#include "absl/status/status.h"
#include "absl/status/statusor.h"
#include "asset_stream_manager/asset_stream_config.h"
#include "asset_stream_manager/base_command.h"
@@ -34,9 +38,8 @@ class StartServiceCommand : public BaseCommand {
absl::Status Run() override;
private:
// Initializes LOG* logging.
// Depending on the flags, might log to console or to a file.
absl::Status InitLogging();
// Depending on the flags, returns a console or file logger.
absl::StatusOr<std::unique_ptr<Log>> GetLogger();
// Runs the asset streaming service.
absl::Status RunService();

75
asset_stream_manager/stop_command.cc Normal file
View File

@@ -0,0 +1,75 @@
// Copyright 2022 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "asset_stream_manager/stop_command.h"
#include <memory>
#include "asset_stream_manager/local_assets_stream_manager_client.h"
#include "asset_stream_manager/session_management_server.h"
#include "common/log.h"
#include "common/path.h"
#include "common/status_macros.h"
#include "lyra/lyra.hpp"
namespace cdc_ft {
namespace {
constexpr int kDefaultVerbosity = 2;
} // namespace
StopCommand::StopCommand(int* exit_code)
: BaseCommand("stop", "Stops a streaming session", exit_code) {}
StopCommand::~StopCommand() = default;
void StopCommand::RegisterCommandLineFlags(lyra::command& cmd) {
verbosity_ = kDefaultVerbosity;
cmd.add_argument(lyra::opt(verbosity_, "num")
.name("--verbosity")
.help("Verbosity of the log output, default: " +
std::to_string(kDefaultVerbosity) +
". Increase to make logs more verbose."));
service_port_ = SessionManagementServer::kDefaultServicePort;
cmd.add_argument(
lyra::opt(service_port_, "port")
.name("--service-port")
.help("Local port to use while connecting to the local "
"asset stream service, default: " +
std::to_string(SessionManagementServer::kDefaultServicePort)));
cmd.add_argument(
lyra::arg(PosArgValidator(&user_host_dir_), "[user@]host:src-dir")
.required()
.help("Linux host and directory to stream to"));
}
absl::Status StopCommand::Run() {
LogLevel level = Log::VerbosityToLogLevel(verbosity_);
ScopedLog scoped_log(std::make_unique<ConsoleLog>(level));
LocalAssetsStreamManagerClient client(service_port_);
std::string user_host, mount_dir;
RETURN_IF_ERROR(LocalAssetsStreamManagerClient::ParseUserHostDir(
user_host_dir_, &user_host, &mount_dir));
absl::Status status = client.StopSession(user_host, mount_dir);
if (status.ok()) {
LOG_INFO("Stopped streaming session to '%s:%s'", user_host, mount_dir);
}
return status;
}
} // namespace cdc_ft

44
asset_stream_manager/stop_command.h Normal file
View File

@@ -0,0 +1,44 @@
/*
* Copyright 2022 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef ASSET_STREAM_MANAGER_STOP_COMMAND_H_
#define ASSET_STREAM_MANAGER_STOP_COMMAND_H_
#include "absl/status/status.h"
#include "asset_stream_manager/base_command.h"
namespace cdc_ft {
// Handler for the stop command. Sends an RPC call to the service to stop an
// asset streaming session.
class StopCommand : public BaseCommand {
public:
explicit StopCommand(int* exit_code);
~StopCommand();
// BaseCommand:
void RegisterCommandLineFlags(lyra::command& cmd) override;
absl::Status Run() override;
private:
int verbosity_ = 0;
uint16_t service_port_ = 0;
std::string user_host_dir_;
};
} // namespace cdc_ft
#endif // ASSET_STREAM_MANAGER_STOP_COMMAND_H_

3
cdc_stream/.gitignore vendored
View File

@@ -1,3 +0,0 @@
x64/*
*.log
*.user

28
cdc_stream/BUILD
View File

@@ -1,28 +0,0 @@
load("@rules_cc//cc:defs.bzl", "cc_binary", "cc_library")
package(default_visibility = [
"//:__subpackages__",
])
cc_binary(
name = "cdc_stream",
srcs = ["main.cc"],
deps = [
":local_assets_stream_manager_client",
"//common:log",
"//common:path",
"@com_google_absl//absl/flags:parse",
"@com_google_absl//absl/status",
],
)
cc_library(
name = "local_assets_stream_manager_client",
srcs = ["local_assets_stream_manager_client.cc"],
hdrs = ["local_assets_stream_manager_client.h"],
deps = [
"//common:grpc_status",
"//proto:local_assets_stream_manager_grpc_proto",
"@com_google_absl//absl/status",
],
)

165
cdc_stream/main.cc
View File

@@ -1,165 +0,0 @@
// Copyright 2022 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include <memory>
#include <vector>
#include "absl/flags/flag.h"
#include "absl/flags/parse.h"
#include "absl/flags/usage.h"
#include "absl/status/status.h"
#include "absl/strings/str_split.h"
#include "cdc_stream/local_assets_stream_manager_client.h"
#include "common/log.h"
#include "common/path.h"
#include "grpcpp/channel.h"
#include "grpcpp/security/credentials.h"
ABSL_FLAG(uint16_t, service_port, 44432,
"Port to use while connecting to the local asset stream service.");
ABSL_FLAG(
uint16_t, ssh_port, 22,
"Port to use while connecting to the remote instance being streamed to.");
ABSL_FLAG(std::string, ssh_command, "",
"Path and arguments of ssh command to use, e.g. "
"\"C:\\path\\to\\ssh.exe -F config_file\". Can also be specified by "
"the CDC_SSH_COMMAND environment variable.");
ABSL_FLAG(std::string, scp_command, "",
"Path and arguments of scp command to use, e.g. "
"\"C:\\path\\to\\scp.exe -F config_file\". Can also be specified by "
"the CDC_SCP_COMMAND environment variable.");
ABSL_FLAG(int, verbosity, 2,
"Verbosity of the log output. Increase to make logs more verbose.");
namespace {
constexpr char kHelpText[] = R"!(Stream files from a Windows to a Linux device
Usage: cdc_stream [flags] start windows_dir [user@]host:linux_dir
Streams the Windows directory windows_dir to directory linux_dir on the
Linux host using SSH username user.
cdc_stream [flags] stop [user@]host:linux_dir
Stops the streaming session to the given Linux target.
Type cdc_stream --helpfull for available flags.)!";
// Splits |user_host_dir| = [user@]host:dir up into [user@]host and dir.
// Does not touch Windows drives, e.g. C:\foo.
bool ParseUserHostDir(const std::string& user_host_dir, std::string* user_host,
std::string* dir) {
std::vector<std::string> parts =
absl::StrSplit(user_host_dir, absl::MaxSplits(':', 1));
if (parts.size() < 2 ||
(parts[0].size() == 1 && toupper(parts[0][0]) >= 'A' &&
toupper(parts[0][0]) <= 'Z')) {
LOG_ERROR(
"Failed to parse '%s'. Make sure it is of the form "
"[user@]host:linux_dir.",
user_host_dir);
return false;
}
*user_host = parts[0];
*dir = parts[1];
return true;
}
std::string GetFlagFromEnvOrArgs(const char* env,
const absl::Flag<std::string>& flag) {
std::string value = absl::GetFlag(flag);
if (value.empty()) {
cdc_ft::path::GetEnv(env, &value).IgnoreError();
}
return value;
}
} // namespace
int main(int argc, char* argv[]) {
absl::SetProgramUsageMessage(kHelpText);
std::vector<char*> args = absl::ParseCommandLine(argc, argv);
uint16_t service_port = absl::GetFlag(FLAGS_service_port);
int verbosity = absl::GetFlag(FLAGS_verbosity);
std::string ssh_command =
GetFlagFromEnvOrArgs("CDC_SSH_COMMAND", FLAGS_ssh_command);
std::string scp_command =
GetFlagFromEnvOrArgs("CDC_SCP_COMMAND", FLAGS_scp_command);
uint16_t ssh_port = absl::GetFlag(FLAGS_ssh_port);
cdc_ft::LogLevel level = cdc_ft::Log::VerbosityToLogLevel(verbosity);
cdc_ft::Log::Initialize(std::make_unique<cdc_ft::ConsoleLog>(level));
if (args.size() < 2) {
LOG_INFO(kHelpText);
return 0;
}
std::string command = args[1];
if (command != "start" && command != "stop") {
LOG_ERROR("Unknown command '%s'. Must be 'start' or 'stop'.", command);
return 1;
}
// Start a gRpc client.
std::string client_address = absl::StrFormat("localhost:%u", service_port);
std::shared_ptr<grpc::Channel> grpc_channel = grpc::CreateCustomChannel(
client_address, grpc::InsecureChannelCredentials(),
grpc::ChannelArguments());
cdc_ft::LocalAssetsStreamManagerClient client(grpc_channel);
absl::Status status;
if (command == "start") {
if (args.size() < 4) {
LOG_ERROR(
"Command 'start' needs 2 arguments: the Windows directory to stream"
" and the Linux target [user@]host:dir.");
return 1;
}
std::string src_dir = cdc_ft::path::GetFullPath(args[2]);
std::string user_host, mount_dir;
if (!ParseUserHostDir(args[3], &user_host, &mount_dir)) return 1;
status = client.StartSession(src_dir, user_host, ssh_port, mount_dir,
ssh_command, scp_command);
if (status.ok()) {
LOG_INFO("Started streaming directory '%s' to '%s:%s'", src_dir,
user_host, mount_dir);
}
} else /* if (command == "stop") */ {
if (args.size() < 3) {
LOG_ERROR(
"Command 'stop' needs 1 argument: the Linux target "
"[user@]host:linux_dir.");
return 1;
}
std::string user_host, mount_dir;
if (!ParseUserHostDir(args[2], &user_host, &mount_dir)) return 1;
status = client.StopSession(user_host, mount_dir);
if (status.ok()) {
LOG_INFO("Stopped streaming session to '%s:%s'", user_host, mount_dir);
}
}
if (!status.ok()) {
LOG_ERROR("Error: %s", status.ToString());
}
cdc_ft::Log::Shutdown();
return static_cast<int>(status.code());
}

7
common/log.h
View File

@@ -123,6 +123,13 @@ class ConsoleLog : public Log {
absl::Mutex mutex_;
};
// Initializes the log in the constructor and shuts it down on destruction.
class ScopedLog {
public:
ScopedLog(std::unique_ptr<Log> log) { Log::Initialize(std::move(log)); }
~ScopedLog() { Log::Shutdown(); }
};
class FileLog : public Log {
public:
FileLog(LogLevel log_level, const char* path);