mirror of
https://github.com/nestriness/cdc-file-transfer.git
synced 2026-01-30 10:35:37 +02:00
a381541d1b
Switch asset_stream_manager to use Lyra Lyra has a nice simple interface, but a few quirks that we work around, mainly in the BaseCommand class: - It does not support return values from running a command. - It does not support return values from a custom arg parser. - Lyra interprets --bad_arg as positional argument. Fixes #15
269 lines
11 KiB
C++
269 lines
11 KiB
C++
// Copyright 2022 Google LLC
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
#include "asset_stream_manager/asset_stream_config.h"
|
|
|
|
#include <sstream>
|
|
|
|
#include "absl/strings/str_format.h"
|
|
#include "absl/strings/str_join.h"
|
|
#include "absl_helper/jedec_size_flag.h"
|
|
#include "common/buffer.h"
|
|
#include "common/path.h"
|
|
#include "common/status_macros.h"
|
|
#include "data_store/data_provider.h"
|
|
#include "data_store/disk_data_store.h"
|
|
#include "json/json.h"
|
|
#include "lyra/lyra.hpp"
|
|
|
|
namespace cdc_ft {
|
|
namespace {
|
|
constexpr int kDefaultVerbosity = 2;
|
|
constexpr uint32_t kDefaultManifestUpdaterThreads = 4;
|
|
constexpr uint32_t kDefaultFileChangeWaitDurationMs = 500;
|
|
} // namespace
|
|
|
|
AssetStreamConfig::AssetStreamConfig() = default;
|
|
|
|
AssetStreamConfig::~AssetStreamConfig() = default;
|
|
|
|
void AssetStreamConfig::RegisterCommandLineFlags(lyra::command& cmd) {
|
|
session_cfg_.verbosity = kDefaultVerbosity;
|
|
cmd.add_argument(lyra::opt(session_cfg_.verbosity, "num")
|
|
.name("--verbosity")
|
|
.help("Verbosity of the log output, default: " +
|
|
std::to_string(kDefaultVerbosity)));
|
|
|
|
cmd.add_argument(
|
|
lyra::opt(session_cfg_.stats)
|
|
.name("--stats")
|
|
.help("Collect and print detailed streaming statistics"));
|
|
|
|
cmd.add_argument(
|
|
lyra::opt(session_cfg_.quiet)
|
|
.name("--quiet")
|
|
.help("Do not print any output except errors and stats"));
|
|
|
|
session_cfg_.manifest_updater_threads = kDefaultManifestUpdaterThreads;
|
|
cmd.add_argument(lyra::opt(session_cfg_.manifest_updater_threads, "count")
|
|
.name("--manifest-updater-threads")
|
|
.help("Number of threads used to compute file hashes on "
|
|
"the workstation, default: " +
|
|
std::to_string(kDefaultManifestUpdaterThreads)));
|
|
|
|
session_cfg_.file_change_wait_duration_ms = kDefaultFileChangeWaitDurationMs;
|
|
cmd.add_argument(
|
|
lyra::opt(session_cfg_.file_change_wait_duration_ms, "ms")
|
|
.name("--file-change-wait-duration-ms")
|
|
.help("Time in milliseconds to wait until pushing a file change "
|
|
"to the instance after detecting it, default: " +
|
|
std::to_string(kDefaultFileChangeWaitDurationMs)));
|
|
|
|
cmd.add_argument(lyra::opt(session_cfg_.fuse_debug)
|
|
.name("--debug")
|
|
.help("Run FUSE filesystem in debug mode"));
|
|
|
|
cmd.add_argument(lyra::opt(session_cfg_.fuse_singlethreaded)
|
|
.name("--singlethreaded")
|
|
.optional()
|
|
.help("Run FUSE filesystem in single-threaded mode"));
|
|
|
|
cmd.add_argument(lyra::opt(session_cfg_.fuse_check)
|
|
.name("--check")
|
|
.help("Check FUSE consistency and log check results"));
|
|
|
|
session_cfg_.fuse_cache_capacity = DiskDataStore::kDefaultCapacity;
|
|
cmd.add_argument(lyra::opt(JedecParser("--cache-capacity",
|
|
&session_cfg_.fuse_cache_capacity),
|
|
"bytes")
|
|
.name("--cache-capacity")
|
|
.help("FUSE cache capacity, default: " +
|
|
std::to_string(DiskDataStore::kDefaultCapacity) +
|
|
". Supports common unit suffixes K, M, G."));
|
|
|
|
session_cfg_.fuse_cleanup_timeout_sec = DataProvider::kCleanupTimeoutSec;
|
|
cmd.add_argument(
|
|
lyra::opt(session_cfg_.fuse_cleanup_timeout_sec, "sec")
|
|
.name("--cleanup-timeout")
|
|
.help("Period in seconds at which instance cache cleanups are run, "
|
|
"default: " +
|
|
std::to_string(DataProvider::kCleanupTimeoutSec)));
|
|
|
|
session_cfg_.fuse_access_idle_timeout_sec = DataProvider::kAccessIdleSec;
|
|
cmd.add_argument(
|
|
lyra::opt(session_cfg_.fuse_access_idle_timeout_sec, "sec")
|
|
.name("--access-idle-timeout")
|
|
.help("Do not run instance cache cleanups for this long after the "
|
|
"last file access, default: " +
|
|
std::to_string(DataProvider::kAccessIdleSec)));
|
|
|
|
cmd.add_argument(lyra::opt(log_to_stdout_)
|
|
.name("--log-to-stdout")
|
|
.help("Log to stdout instead of to a file"));
|
|
cmd.add_argument(
|
|
lyra::opt(dev_src_dir_, "dir")
|
|
.name("--dev-src-dir")
|
|
.help("Start a streaming session immediately from the given Windows "
|
|
"path. Used during development. Must also specify other --dev "
|
|
"flags."));
|
|
|
|
cmd.add_argument(
|
|
lyra::opt(dev_target_.user_host, "[user@]host")
|
|
.name("--dev-user-host")
|
|
.help("Username and host to stream to. See also --dev-src-dir."));
|
|
|
|
dev_target_.ssh_port = RemoteUtil::kDefaultSshPort;
|
|
cmd.add_argument(
|
|
lyra::opt(dev_target_.ssh_port, "port")
|
|
.name("--dev-ssh-port")
|
|
.help("SSH port to use for the connection to the host, default: " +
|
|
std::to_string(RemoteUtil::kDefaultSshPort) +
|
|
". See also --dev-src-dir."));
|
|
|
|
cmd.add_argument(
|
|
lyra::opt(dev_target_.ssh_command, "cmd")
|
|
.name("--dev-ssh-command")
|
|
.help("Ssh command and extra flags to use for the "
|
|
"connection to the host. See also --dev-src-dir."));
|
|
|
|
cmd.add_argument(
|
|
lyra::opt(dev_target_.scp_command, "cmd")
|
|
.name("--dev-scp-command")
|
|
.help("Scp command and extra flags to use for the "
|
|
"connection to the host. See also --dev-src-dir."));
|
|
|
|
cmd.add_argument(
|
|
lyra::opt(dev_target_.mount_dir, "dir")
|
|
.name("--dev-mount-dir")
|
|
.help("Directory on the host to stream to. See also --dev-src-dir."));
|
|
}
|
|
|
|
absl::Status AssetStreamConfig::LoadFromFile(const std::string& path) {
|
|
Buffer buffer;
|
|
RETURN_IF_ERROR(path::ReadFile(path, &buffer));
|
|
|
|
Json::Value config;
|
|
Json::Reader reader;
|
|
if (!reader.parse(buffer.data(), buffer.data() + buffer.size(), config,
|
|
false)) {
|
|
return absl::InvalidArgumentError(
|
|
absl::StrFormat("Failed to parse config file '%s': %s", path,
|
|
reader.getFormattedErrorMessages()));
|
|
}
|
|
|
|
#define ASSIGN_VAR(var, flag, type) \
|
|
do { \
|
|
if (config.isMember(flag)) { \
|
|
var = config[flag].as##type(); \
|
|
flags_read_from_file_.insert(flag); \
|
|
} \
|
|
} while (0)
|
|
|
|
ASSIGN_VAR(session_cfg_.verbosity, "verbosity", Int);
|
|
ASSIGN_VAR(session_cfg_.fuse_debug, "debug", Bool);
|
|
ASSIGN_VAR(session_cfg_.fuse_singlethreaded, "singlethreaded", Bool);
|
|
ASSIGN_VAR(session_cfg_.stats, "stats", Bool);
|
|
ASSIGN_VAR(session_cfg_.quiet, "quiet", Bool);
|
|
ASSIGN_VAR(session_cfg_.fuse_check, "check", Bool);
|
|
ASSIGN_VAR(log_to_stdout_, "log-to-stdout", Bool);
|
|
ASSIGN_VAR(session_cfg_.fuse_cleanup_timeout_sec, "cleanup-timeout", Int);
|
|
ASSIGN_VAR(session_cfg_.fuse_access_idle_timeout_sec, "access-idle-timeout",
|
|
Int);
|
|
ASSIGN_VAR(session_cfg_.manifest_updater_threads, "manifest-updater-threads",
|
|
Int);
|
|
ASSIGN_VAR(session_cfg_.file_change_wait_duration_ms,
|
|
"file-change-wait-duration-ms", Int);
|
|
|
|
// cache_capacity requires Jedec size conversion.
|
|
constexpr char kCacheCapacity[] = "cache-capacity";
|
|
if (config.isMember(kCacheCapacity)) {
|
|
JedecSize cache_capacity;
|
|
std::string error;
|
|
if (AbslParseFlag(config[kCacheCapacity].asString(), &cache_capacity,
|
|
&error)) {
|
|
session_cfg_.fuse_cache_capacity = cache_capacity.Size();
|
|
flags_read_from_file_.insert(kCacheCapacity);
|
|
} else {
|
|
// Note that |error| can't be logged here since this code runs before
|
|
// logging is initialized.
|
|
flag_read_errors_[kCacheCapacity] = error;
|
|
}
|
|
}
|
|
|
|
#undef ASSIGN_VAR
|
|
|
|
return absl::OkStatus();
|
|
} // namespace cdc_ft
|
|
|
|
std::string AssetStreamConfig::ToString() {
|
|
std::ostringstream ss;
|
|
ss << "verbosity = " << session_cfg_.verbosity
|
|
<< std::endl;
|
|
ss << "debug = " << session_cfg_.fuse_debug
|
|
<< std::endl;
|
|
ss << "singlethreaded = " << session_cfg_.fuse_singlethreaded
|
|
<< std::endl;
|
|
ss << "stats = " << session_cfg_.stats << std::endl;
|
|
ss << "quiet = " << session_cfg_.quiet << std::endl;
|
|
ss << "check = " << session_cfg_.fuse_check
|
|
<< std::endl;
|
|
ss << "log-to-stdout = " << log_to_stdout_ << std::endl;
|
|
ss << "cache-capacity = " << session_cfg_.fuse_cache_capacity
|
|
<< std::endl;
|
|
ss << "cleanup-timeout = "
|
|
<< session_cfg_.fuse_cleanup_timeout_sec << std::endl;
|
|
ss << "access-idle-timeout = "
|
|
<< session_cfg_.fuse_access_idle_timeout_sec << std::endl;
|
|
ss << "manifest-updater-threads = "
|
|
<< session_cfg_.manifest_updater_threads << std::endl;
|
|
ss << "file-change-wait-duration-ms = "
|
|
<< session_cfg_.file_change_wait_duration_ms << std::endl;
|
|
ss << "dev-src-dir = " << dev_src_dir_ << std::endl;
|
|
ss << "dev-user-host = " << dev_target_.user_host << std::endl;
|
|
ss << "dev-ssh-port = " << dev_target_.ssh_port << std::endl;
|
|
ss << "dev-ssh-command = " << dev_target_.ssh_command
|
|
<< std::endl;
|
|
ss << "dev-scp-command = " << dev_target_.scp_command
|
|
<< std::endl;
|
|
ss << "dev-mount-dir = " << dev_target_.mount_dir << std::endl;
|
|
return ss.str();
|
|
}
|
|
|
|
std::string AssetStreamConfig::GetFlagsReadFromFile() {
|
|
return absl::StrJoin(flags_read_from_file_, ", ");
|
|
}
|
|
|
|
std::string AssetStreamConfig::GetFlagReadErrors() {
|
|
std::string error_str;
|
|
for (const auto& [flag, error] : flag_read_errors_)
|
|
error_str += absl::StrFormat("%sFailed to read '%s': %s",
|
|
error_str.empty() ? "" : "\n", flag, error);
|
|
return error_str;
|
|
}
|
|
std::function<void(const std::string&)> AssetStreamConfig::JedecParser(
|
|
const char* flag_name, uint64_t* bytes) {
|
|
return [flag_name, bytes,
|
|
error = &jedec_parse_error_](const std::string& value) {
|
|
JedecSize size;
|
|
if (AbslParseFlag(value, &size, error)) {
|
|
*bytes = size.Size();
|
|
} else {
|
|
*error = absl::StrFormat("Failed to parse --%s=%s: %s", flag_name, value,
|
|
*error);
|
|
}
|
|
};
|
|
}
|
|
|
|
} // namespace cdc_ft
|