github-mirrors/netris-cdc-file-transfer
1
0
Fork 0
mirror of https://github.com/nestriness/cdc-file-transfer.git synced 2026-03-17 03:43:07 +02:00
Code Issues Packages Projects Releases Wiki Activity

[cdc_stream] [cdc_rsync] Add --forward-port flag (#45)

Browse Source 
Adds a flag to set the SSH forwarding port or port range used for
'cdc_stream start-service' and 'cdc_rsync'.

If a single number is passed, e.g. --forward-port 12345, then this
port is used without checking availability of local and remote ports.
If the port is taken, this results in an error when trying to connect.
Note that this restricts the number of connections that stream can
make to one.

If a range is passed, e.g. --forward-port 45000-46000, the tools
search for available ports locally and remotely in that range. This is
more robust, but a bit slower due to the extra overhead.

Optimizes port_manager_win as it was very slow for a large port range.
It's still not optimal, but the time needed to scan 30k ports is
<< 1 seconds now.

Fixes #12
This commit is contained in:
Lutz Justen
2022-12-19 10:04:36 +01:00
committed by GitHub
parent f8438aec66
commit d8c2b5906e
25 changed files with 419 additions and 164 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
cdc_stream/BUILD
View File

@@ -24,6 +24,7 @@ cc_library(
hdrs = ["base_command.h"],
deps = [
"//absl_helper:jedec_size_flag",
"//common:port_range_parser",
"@com_github_lyra//:lyra",
"@com_google_absl//absl/status",
"@com_google_absl//absl/strings:str_format",

19
cdc_stream/asset_stream_config.cc
View File

@@ -20,6 +20,7 @@
#include "absl/strings/str_join.h"
#include "absl_helper/jedec_size_flag.h"
#include "cdc_stream/base_command.h"
#include "cdc_stream/multi_session.h"
#include "cdc_stream/session_management_server.h"
#include "common/buffer.h"
#include "common/path.h"
@@ -49,6 +50,20 @@ void AssetStreamConfig::RegisterCommandLineFlags(lyra::command& cmd,
"asset stream service, default: " +
std::to_string(service_port_)));
session_cfg_.forward_port_first = MultiSession::kDefaultForwardPortFirst;
session_cfg_.forward_port_last = MultiSession::kDefaultForwardPortLast;
cmd.add_argument(
lyra::opt(base_command.PortRangeParser("--forward-port",
&session_cfg_.forward_port_first,
&session_cfg_.forward_port_last),
"port")
.name("--forward-port")
.help("TCP port or range used for SSH port forwarding, default: " +
std::to_string(MultiSession::kDefaultForwardPortFirst) + "-" +
std::to_string(MultiSession::kDefaultForwardPortLast) +
". If a range is specified, searches for available ports "
"(slower)."));
session_cfg_.verbosity = kDefaultVerbosity;
cmd.add_argument(lyra::opt(session_cfg_.verbosity, "num")
.name("--verbosity")
@@ -175,6 +190,8 @@ absl::Status AssetStreamConfig::LoadFromFile(const std::string& path) {
} while (0)
ASSIGN_VAR(service_port_, "service-port", Int);
ASSIGN_VAR(session_cfg_.forward_port_first, "forward-port-first", Int);
ASSIGN_VAR(session_cfg_.forward_port_last, "forward-port-last", Int);
ASSIGN_VAR(session_cfg_.verbosity, "verbosity", Int);
ASSIGN_VAR(session_cfg_.fuse_debug, "debug", Bool);
ASSIGN_VAR(session_cfg_.fuse_singlethreaded, "singlethreaded", Bool);
@@ -214,6 +231,8 @@ absl::Status AssetStreamConfig::LoadFromFile(const std::string& path) {
std::string AssetStreamConfig::ToString() {
std::ostringstream ss;
ss << "service-port = " << service_port_ << std::endl;
ss << "forward-port = " << session_cfg_.forward_port_first
<< "-" << session_cfg_.forward_port_last << std::endl;
ss << "verbosity = " << session_cfg_.verbosity
<< std::endl;
ss << "debug = " << session_cfg_.fuse_debug

15
cdc_stream/asset_stream_config.h
View File

@@ -48,18 +48,21 @@ class AssetStreamConfig {
// Loads a configuration from the JSON file at |path| and overrides any config
// values that are set in this file. Sample json file:
// {
// "service-port":44432
// "forward-port-first":"44433"
// "forward-port-last":"44442"
// "verbosity":3,
// "debug":0,
// "singlethreaded":0,
// "stats":0,
// "quiet":0,
// "check":0,
// "log_to_stdout":0,
// "cache_capacity":"150G",
// "cleanup_timeout":300,
// "access_idle_timeout":5,
// "manifest_updater_threads":4,
// "file_change_wait_duration_ms":500
// "log-to-stdout":0,
// "cache-capacity":"150G",
// "cleanup-timeout":300,
// "access-idle-timeout":5,
// "manifest-updater-threads":4,
// "file-change-wait-duration-ms":500
// }
// Returns NotFoundError if the file does not exist.
// Returns InvalidArgumentError if the file is not valid JSON.

21
cdc_stream/base_command.cc
View File

@@ -15,7 +15,9 @@
#include "cdc_stream/base_command.h"
#include "absl/strings/str_format.h"
#include "absl/strings/str_split.h"
#include "absl_helper/jedec_size_flag.h"
#include "common/port_range_parser.h"
#include "lyra/lyra.hpp"
namespace cdc_ft {
@@ -44,8 +46,7 @@ void BaseCommand::Register(lyra::cli& cli) {
std::function<void(const std::string&)> BaseCommand::JedecParser(
const char* flag_name, uint64_t* bytes) {
return [flag_name, bytes,
error = &jedec_parse_error_](const std::string& value) {
return [flag_name, bytes, error = &parse_error_](const std::string& value) {
JedecSize size;
if (AbslParseFlag(value, &size, error)) {
*bytes = size.Size();
@@ -56,6 +57,18 @@ std::function<void(const std::string&)> BaseCommand::JedecParser(
};
}
std::function<void(const std::string&)> BaseCommand::PortRangeParser(
const char* flag_name, uint16_t* first, uint16_t* last) {
return [flag_name, first, last,
error = &parse_error_](const std::string& value) {
if (!port_range::Parse(value.c_str(), first, last)) {
*error = absl::StrFormat(
"Failed to parse %s=%s, expected <port> or <port1>-<port2>",
flag_name, value);
}
};
}
std::function<void(const std::string&)> BaseCommand::PosArgValidator(
std::string* str) {
return [str, invalid_arg = &invalid_arg_](const std::string& value) {
@@ -83,8 +96,8 @@ void BaseCommand::CommandHandler(const lyra::group& g) {
return;
}
if (!jedec_parse_error_.empty()) {
std::cerr << "Error: " << jedec_parse_error_ << std::endl;
if (!parse_error_.empty()) {
std::cerr << "Error: " << parse_error_ << std::endl;
*exit_code_ = 1;
return;
}

11
cdc_stream/base_command.h
View File

@@ -48,6 +48,13 @@ class BaseCommand {
std::function<void(const std::string&)> JedecParser(const char* flag_name,
uint64_t* bytes);
// Parser for single ports "123" or port ranges "123-234". Usage:
// lyra::opt(PortRangeParser("port-flag", &first, &last), "port"))
// Automatically reports a parse failure on error.
std::function<void(const std::string&)> PortRangeParser(const char* flag_name,
uint16_t* first,
uint16_t* last);
// Validator that should be used for all positional arguments. Lyra interprets
// -u, --unknown_flag as positional argument. This validator makes sure that
// a positional argument starting with - is reported as an error. Otherwise,
@@ -82,9 +89,9 @@ class BaseCommand {
// Extraneous positional args. Gets reported as error if present.
std::string extra_positional_arg_;
// Errors from parsing JEDEC sizes.
// Errors from custom flag parsers, e.g. JEDEC sizes or port ranges.
// Works around Lyra not accepting errors from parsers.
std::string jedec_parse_error_;
std::string parse_error_;
};
} // namespace cdc_ft

2
cdc_stream/cdc_stream.vcxproj
View File

@@ -47,7 +47,7 @@
<AdditionalOptions>/std:c++17</AdditionalOptions>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
<OutDir>$(SolutionDir)bazel-out\x64_windows-opt\bin\asset_stcdc_streamream_manager\</OutDir>
<OutDir>$(SolutionDir)bazel-out\x64_windows-opt\bin\cdc_stream\</OutDir>
<NMakePreprocessorDefinitions>UNICODE</NMakePreprocessorDefinitions>
<AdditionalOptions>/std:c++17</AdditionalOptions>
</PropertyGroup>

31
cdc_stream/multi_session.cc
View File

@@ -34,11 +34,6 @@
namespace cdc_ft {
namespace {
// Ports used by the asset streaming service for local port forwarding on
// workstation and gamelet.
constexpr int kAssetStreamPortFirst = 44433;
constexpr int kAssetStreamPortLast = 44442;
// Stats output period (if enabled).
constexpr double kStatsPrintDelaySec = 0.1f;
@@ -441,16 +436,19 @@ absl::Status MultiSession::Initialize() {
}
// Find an available local port.
std::unordered_set<int> ports;
ASSIGN_OR_RETURN(
ports,
PortManager::FindAvailableLocalPorts(kAssetStreamPortFirst,
kAssetStreamPortLast, "127.0.0.1",
process_factory_),
"Failed to find an available local port in the range [%d, %d]",
kAssetStreamPortFirst, kAssetStreamPortLast);
assert(!ports.empty());
local_asset_stream_port_ = *ports.begin();
local_asset_stream_port_ = cfg_.forward_port_first;
if (cfg_.forward_port_first < cfg_.forward_port_last) {
std::unordered_set<int> ports;
ASSIGN_OR_RETURN(
ports,
PortManager::FindAvailableLocalPorts(cfg_.forward_port_first,
cfg_.forward_port_last,
"127.0.0.1", process_factory_),
"Failed to find an available local port in the range [%d, %d]",
cfg_.forward_port_first, cfg_.forward_port_last);
assert(!ports.empty());
local_asset_stream_port_ = *ports.begin();
}
assert(!runner_);
runner_ = std::make_unique<MultiSessionRunner>(
@@ -526,7 +524,8 @@ absl::Status MultiSession::StartSession(const std::string& instance_id,
auto session = std::make_unique<Session>(
instance_id, target, cfg_, process_factory_, std::move(metrics_recorder));
RETURN_IF_ERROR(session->Start(local_asset_stream_port_,
kAssetStreamPortFirst, kAssetStreamPortLast));
cfg_.forward_port_first,
cfg_.forward_port_last));
// Wait for the FUSE to receive the first intermediate manifest.
RETURN_IF_ERROR(runner_->WaitForManifestAck(instance_id, absl::Seconds(5)));

5
cdc_stream/multi_session.h
View File

@@ -134,6 +134,11 @@ class MultiSessionRunner {
// to an arbitrary number of gamelets.
class MultiSession {
public:
// Ports used by the asset streaming service for local port forwarding on
// workstation and gamelet.
static constexpr int kDefaultForwardPortFirst = 44433;
static constexpr int kDefaultForwardPortLast = 44442;
// Maximum length of cache path. We must be able to write content hashes into
// this path:
// <cache path>\01234567890123456789<null terminator> = 260 characters.

23
cdc_stream/session.cc
View File

@@ -71,16 +71,19 @@ Session::~Session() {
absl::Status Session::Start(int local_port, int first_remote_port,
int last_remote_port) {
// Find an available remote port.
std::unordered_set<int> ports;
ASSIGN_OR_RETURN(
ports,
PortManager::FindAvailableRemotePorts(
first_remote_port, last_remote_port, "127.0.0.1", process_factory_,
&remote_util_, kInstanceConnectionTimeoutSec),
"Failed to find an available remote port in the range [%d, %d]",
first_remote_port, last_remote_port);
assert(!ports.empty());
int remote_port = *ports.begin();
int remote_port = first_remote_port;
if (first_remote_port < last_remote_port) {
std::unordered_set<int> ports;
ASSIGN_OR_RETURN(
ports,
PortManager::FindAvailableRemotePorts(
first_remote_port, last_remote_port, "127.0.0.1", process_factory_,
&remote_util_, kInstanceConnectionTimeoutSec),
"Failed to find an available remote port in the range [%d, %d]",
first_remote_port, last_remote_port);
assert(!ports.empty());
remote_port = *ports.begin();
}
assert(!fuse_);
fuse_ = std::make_unique<CdcFuseManager>(instance_id_, process_factory_,

4
cdc_stream/session_config.h
View File

@@ -56,6 +56,10 @@ struct SessionConfig {
// Time to wait until running a manifest update after detecting a file change.
uint32_t file_change_wait_duration_ms = 0;
// Ports used for local port forwarding.
uint16_t forward_port_first = 0;
uint16_t forward_port_last = 0;
};
} // namespace cdc_ft

1
cdc_stream/start_command.cc
View File

@@ -22,7 +22,6 @@
#include "common/log.h"
#include "common/path.h"
#include "common/process.h"
#include "common/remote_util.h"
#include "common/status_macros.h"
#include "common/stopwatch.h"
#include "common/util.h"