github-mirrors/netris-cdc-file-transfer
1
0
Fork 0
mirror of https://github.com/nestriness/cdc-file-transfer.git synced 2026-01-30 14:45:37 +02:00
Code Issues Packages Projects Releases Wiki Activity

[cdc_stream] [cdc_rsync] Add --forward-port flag (#45)

Browse Source 
Adds a flag to set the SSH forwarding port or port range used for
'cdc_stream start-service' and 'cdc_rsync'.

If a single number is passed, e.g. --forward-port 12345, then this
port is used without checking availability of local and remote ports.
If the port is taken, this results in an error when trying to connect.
Note that this restricts the number of connections that stream can
make to one.

If a range is passed, e.g. --forward-port 45000-46000, the tools
search for available ports locally and remotely in that range. This is
more robust, but a bit slower due to the extra overhead.

Optimizes port_manager_win as it was very slow for a large port range.
It's still not optimal, but the time needed to scan 30k ports is
<< 1 seconds now.

Fixes #12
This commit is contained in:
Lutz Justen
2022-12-19 10:04:36 +01:00
committed by GitHub
parent f8438aec66
commit d8c2b5906e
25 changed files with 419 additions and 164 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
common/BUILD
View File

@@ -254,6 +254,25 @@ cc_test(
],
)
cc_library(
name = "port_range_parser",
srcs = ["port_range_parser.cc"],
hdrs = ["port_range_parser.h"],
deps = [
"@com_google_absl//absl/strings",
],
)
cc_test(
name = "port_range_parser_test",
srcs = ["port_range_parser_test.cc"],
deps = [
":port_range_parser",
":test_main",
"@com_google_googletest//:gtest",
],
)
cc_library(
name = "process",
srcs = ["process_win.cc"],

7
common/port_manager.h
View File

@@ -51,14 +51,11 @@ class PortManager {
// Reserves a port in the range passed to the constructor. The port is
// released automatically upon destruction if ReleasePort() is not called
// explicitly.
// |check_remote| determines whether the remote port should be checked as
// well. If false, the check is skipped and a port might be returned that is
// still in use remotely.
// |remote_timeout_sec| is the timeout for finding available ports on the
// remote instance. Not used if |check_remote| is false.
// remote instance.
// Returns a DeadlineExceeded error if the timeout is exceeded.
// Returns a ResourceExhausted error if no ports are available.
absl::StatusOr<int> ReservePort(bool check_remote, int remote_timeout_sec);
absl::StatusOr<int> ReservePort(int remote_timeout_sec);
// Releases a reserved port.
absl::Status ReleasePort(int port);

57
common/port_manager_test.cc
View File

@@ -38,9 +38,6 @@ constexpr int kTimeoutSec = 1;
constexpr char kLocalNetstat[] = "netstat -a -n -p tcp";
constexpr char kRemoteNetstat[] = "netstat --numeric --listening --tcp";
constexpr bool kCheckRemote = true;
constexpr bool kNoCheckRemote = false;
constexpr char kLocalNetstatOutFmt[] =
"TCP 127.0.0.1:50000 127.0.0.1:%i ESTABLISHED";
constexpr char kRemoteNetstatOutFmt[] =
@@ -73,16 +70,7 @@ TEST_F(PortManagerTest, ReservePortSuccess) {
process_factory_.SetProcessOutput(kLocalNetstat, "", "", 0);
process_factory_.SetProcessOutput(kRemoteNetstat, "", "", 0);
absl::StatusOr<int> port =
port_manager_.ReservePort(kCheckRemote, kTimeoutSec);
ASSERT_OK(port);
EXPECT_EQ(*port, kFirstPort);
}
TEST_F(PortManagerTest, ReservePortNoRemoteSuccess) {
process_factory_.SetProcessOutput(kLocalNetstat, "", "", 0);
absl::StatusOr<int> port = port_manager_.ReservePort(kNoCheckRemote, 0);
absl::StatusOr<int> port = port_manager_.ReservePort(kTimeoutSec);
ASSERT_OK(port);
EXPECT_EQ(*port, kFirstPort);
}
@@ -95,8 +83,7 @@ TEST_F(PortManagerTest, ReservePortAllLocalPortsTaken) {
process_factory_.SetProcessOutput(kLocalNetstat, local_netstat_out, "", 0);
process_factory_.SetProcessOutput(kRemoteNetstat, "", "", 0);
absl::StatusOr<int> port =
port_manager_.ReservePort(kCheckRemote, kTimeoutSec);
absl::StatusOr<int> port = port_manager_.ReservePort(kTimeoutSec);
EXPECT_TRUE(absl::IsResourceExhausted(port.status()));
EXPECT_TRUE(
absl::StrContains(port.status().message(), "No port available in range"));
@@ -110,8 +97,7 @@ TEST_F(PortManagerTest, ReservePortAllRemotePortsTaken) {
process_factory_.SetProcessOutput(kLocalNetstat, "", "", 0);
process_factory_.SetProcessOutput(kRemoteNetstat, remote_netstat_out, "", 0);
absl::StatusOr<int> port =
port_manager_.ReservePort(kCheckRemote, kTimeoutSec);
absl::StatusOr<int> port = port_manager_.ReservePort(kTimeoutSec);
EXPECT_TRUE(absl::IsResourceExhausted(port.status()));
EXPECT_TRUE(
absl::StrContains(port.status().message(), "No port available in range"));
@@ -121,8 +107,7 @@ TEST_F(PortManagerTest, ReservePortLocalNetstatFails) {
process_factory_.SetProcessOutput(kLocalNetstat, "", "", 1);
process_factory_.SetProcessOutput(kRemoteNetstat, "", "", 0);
absl::StatusOr<int> port =
port_manager_.ReservePort(kCheckRemote, kTimeoutSec);
absl::StatusOr<int> port = port_manager_.ReservePort(kTimeoutSec);
EXPECT_NOT_OK(port);
EXPECT_TRUE(
absl::StrContains(port.status().message(),
@@ -133,8 +118,7 @@ TEST_F(PortManagerTest, ReservePortRemoteNetstatFails) {
process_factory_.SetProcessOutput(kLocalNetstat, "", "", 0);
process_factory_.SetProcessOutput(kRemoteNetstat, "", "", 1);
absl::StatusOr<int> port =
port_manager_.ReservePort(kCheckRemote, kTimeoutSec);
absl::StatusOr<int> port = port_manager_.ReservePort(kTimeoutSec);
EXPECT_NOT_OK(port);
EXPECT_TRUE(absl::StrContains(port.status().message(),
"Failed to find available ports on instance"));
@@ -145,8 +129,7 @@ TEST_F(PortManagerTest, ReservePortRemoteNetstatTimesOut) {
process_factory_.SetProcessNeverExits(kRemoteNetstat);
steady_clock_.AutoAdvance(kTimeoutSec * 2 * 1000);
absl::StatusOr<int> port =
port_manager_.ReservePort(kCheckRemote, kTimeoutSec);
absl::StatusOr<int> port = port_manager_.ReservePort(kTimeoutSec);
EXPECT_NOT_OK(port);
EXPECT_TRUE(absl::IsDeadlineExceeded(port.status()));
EXPECT_TRUE(absl::StrContains(port.status().message(),
@@ -163,14 +146,10 @@ TEST_F(PortManagerTest, ReservePortMultipleInstances) {
// Port managers use shared memory, so different instances know about each
// other. This would even work if |port_manager_| and |port_manager2| belonged
// to different processes, but we don't test that here.
EXPECT_EQ(*port_manager_.ReservePort(kCheckRemote, kTimeoutSec),
kFirstPort + 0);
EXPECT_EQ(*port_manager2.ReservePort(kCheckRemote, kTimeoutSec),
kFirstPort + 1);
EXPECT_EQ(*port_manager_.ReservePort(kCheckRemote, kTimeoutSec),
kFirstPort + 2);
EXPECT_EQ(*port_manager2.ReservePort(kCheckRemote, kTimeoutSec),
kFirstPort + 3);
EXPECT_EQ(*port_manager_.ReservePort(kTimeoutSec), kFirstPort + 0);
EXPECT_EQ(*port_manager2.ReservePort(kTimeoutSec), kFirstPort + 1);
EXPECT_EQ(*port_manager_.ReservePort(kTimeoutSec), kFirstPort + 2);
EXPECT_EQ(*port_manager2.ReservePort(kTimeoutSec), kFirstPort + 3);
}
TEST_F(PortManagerTest, ReservePortReusesPortsInLRUOrder) {
@@ -178,7 +157,7 @@ TEST_F(PortManagerTest, ReservePortReusesPortsInLRUOrder) {
process_factory_.SetProcessOutput(kRemoteNetstat, "", "", 0);
for (int n = 0; n < kNumPorts * 2; ++n) {
EXPECT_EQ(*port_manager_.ReservePort(kCheckRemote, kTimeoutSec),
EXPECT_EQ(*port_manager_.ReservePort(kTimeoutSec),
kFirstPort + n % kNumPorts);
system_clock_.Advance(1000);
}
@@ -188,11 +167,10 @@ TEST_F(PortManagerTest, ReleasePort) {
process_factory_.SetProcessOutput(kLocalNetstat, "", "", 0);
process_factory_.SetProcessOutput(kRemoteNetstat, "", "", 0);
absl::StatusOr<int> port =
port_manager_.ReservePort(kCheckRemote, kTimeoutSec);
absl::StatusOr<int> port = port_manager_.ReservePort(kTimeoutSec);
EXPECT_EQ(*port, kFirstPort);
EXPECT_OK(port_manager_.ReleasePort(*port));
port = port_manager_.ReservePort(kCheckRemote, kTimeoutSec);
port = port_manager_.ReservePort(kTimeoutSec);
EXPECT_EQ(*port, kFirstPort);
}
@@ -202,13 +180,10 @@ TEST_F(PortManagerTest, ReleasePortOnDestruction) {
auto port_manager2 = std::make_unique<PortManager>(
kGuid, kFirstPort, kLastPort, &process_factory_, &remote_util_);
EXPECT_EQ(*port_manager2->ReservePort(kCheckRemote, kTimeoutSec),
kFirstPort + 0);
EXPECT_EQ(*port_manager_.ReservePort(kCheckRemote, kTimeoutSec),
kFirstPort + 1);
EXPECT_EQ(*port_manager2->ReservePort(kTimeoutSec), kFirstPort + 0);
EXPECT_EQ(*port_manager_.ReservePort(kTimeoutSec), kFirstPort + 1);
port_manager2.reset();
EXPECT_EQ(*port_manager_.ReservePort(kCheckRemote, kTimeoutSec),
kFirstPort + 0);
EXPECT_EQ(*port_manager_.ReservePort(kTimeoutSec), kFirstPort + 0);
}
TEST_F(PortManagerTest, FindAvailableLocalPortsSuccess) {

24
common/port_manager_win.cc
View File

@@ -121,8 +121,7 @@ PortManager::~PortManager() {
}
}
absl::StatusOr<int> PortManager::ReservePort(bool check_remote,
int remote_timeout_sec) {
absl::StatusOr<int> PortManager::ReservePort(int remote_timeout_sec) {
// Find available port on workstation.
std::unordered_set<int> local_ports;
ASSIGN_OR_RETURN(local_ports,
@@ -132,13 +131,11 @@ absl::StatusOr<int> PortManager::ReservePort(bool check_remote,
// Find available port on remote instance.
std::unordered_set<int> remote_ports = local_ports;
if (check_remote) {
ASSIGN_OR_RETURN(remote_ports,
FindAvailableRemotePorts(
first_port_, last_port_, "0.0.0.0", process_factory_,
remote_util_, remote_timeout_sec, steady_clock_),
"Failed to find available ports on instance");
}
ASSIGN_OR_RETURN(remote_ports,
FindAvailableRemotePorts(first_port_, last_port_, "0.0.0.0",
process_factory_, remote_util_,
remote_timeout_sec, steady_clock_),
"Failed to find available ports on instance");
// Fetch shared memory.
void* mem;
@@ -290,9 +287,14 @@ absl::StatusOr<std::unordered_set<int>> PortManager::FindAvailablePorts(
int first_port, int last_port, const std::string& netstat_output,
const char* ip) {
std::unordered_set<int> available_ports;
for (int port = first_port; port <= last_port; ++port) {
std::vector<std::string> lines = absl::StrSplit(netstat_output, '\n');
std::vector<std::string> lines;
for (const auto& line : absl::StrSplit(netstat_output, '\n')) {
if (absl::StrContains(line, ip)) {
lines.push_back(std::string(line));
}
}
for (int port = first_port; port <= last_port; ++port) {
bool port_occupied = false;
std::string portToken = absl::StrFormat("%s:%i", ip, port);
for (const std::string& line : lines) {

40
common/port_range_parser.cc Normal file
View File

@@ -0,0 +1,40 @@
// Copyright 2022 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "common/port_range_parser.h"
#include <cassert>
#include "absl/strings/str_split.h"
namespace cdc_ft {
namespace port_range {
bool Parse(const char* value, uint16_t* first, uint16_t* last) {
assert(value);
*first = 0;
*last = 0;
std::vector<std::string> parts = absl::StrSplit(value, '-');
if (parts.empty() || parts.size() > 2) return false;
const int ifirst = atoi(parts[0].c_str());
const int ilast = parts.size() > 1 ? atoi(parts[1].c_str()) : ifirst;
if (ifirst <= 0 || ifirst > UINT16_MAX) return false;
if (ilast <= 0 || ilast > UINT16_MAX || ifirst > ilast) return false;
*first = static_cast<uint16_t>(ifirst);
*last = static_cast<uint16_t>(ilast);
return true;
}
} // namespace port_range
} // namespace cdc_ft

33
common/port_range_parser.h Normal file
View File

@@ -0,0 +1,33 @@
/*
* Copyright 2022 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef COMMON_PORT_RANGE_PARSER_H_
#define COMMON_PORT_RANGE_PARSER_H_
#include <cstdint>
namespace cdc_ft {
namespace port_range {
// Parses |value| into a port range |first|-|last|.
// If |value| is a single number a, assigns |first|=|last|=a.
// If |value| is a range a-b, assigns |first|=a, |last|=b.
bool Parse(const char* value, uint16_t* first, uint16_t* last);
} // namespace port_range
} // namespace cdc_ft
#endif // COMMON_PORT_RANGE_PARSER_H_

69
common/port_range_parser_test.cc Normal file
View File

@@ -0,0 +1,69 @@
// Copyright 2022 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "common/port_range_parser.h"
#include "gtest/gtest.h"
namespace cdc_ft {
namespace {
TEST(PortRangeParserTest, SingleSuccess) {
uint16_t first, last;
EXPECT_TRUE(port_range::Parse("65535", &first, &last));
EXPECT_EQ(first, 65535);
EXPECT_EQ(last, 65535);
}
TEST(PortRangeParserTest, RangeSuccess) {
uint16_t first, last;
EXPECT_TRUE(port_range::Parse("1-2", &first, &last));
EXPECT_EQ(first, 1);
EXPECT_EQ(last, 2);
}
TEST(ParamsTest, NoValueFail) {
uint16_t first = 1, last = 1;
EXPECT_FALSE(port_range::Parse("", &first, &last));
EXPECT_EQ(first, 0);
EXPECT_EQ(last, 0);
}
TEST(ParamsTest, BadValueTooSmallFail) {
uint16_t first, last;
EXPECT_FALSE(port_range::Parse("0", &first, &last));
}
TEST(ParamsTest, BadValueNotIntegerFail) {
uint16_t first, last;
EXPECT_FALSE(port_range::Parse("port", &first, &last));
}
TEST(ParamsTest, ForwardPort_BadRangeTooBig) {
uint16_t first, last;
EXPECT_FALSE(port_range::Parse("50000-65536", &first, &last));
}
TEST(ParamsTest, ForwardPort_BadRangeFirstGtLast) {
uint16_t first, last;
EXPECT_FALSE(port_range::Parse("50001-50000", &first, &last));
}
TEST(ParamsTest, ForwardPort_BadRangeTwoMinus) {
uint16_t first, last;
EXPECT_FALSE(port_range::Parse("1-2-3", &first, &last));
}
} // namespace
} // namespace cdc_ft