github-mirrors/netris-cdc-file-transfer
1
0
Fork 0
mirror of https://github.com/nestriness/cdc-file-transfer.git synced 2026-05-01 16:43:08 +03:00
Code Issues Packages Projects Releases Wiki Activity

Releasing the former Stadia file transfer tools

Browse Source 
The tools allow efficient and fast synchronization of large directory
trees from a Windows workstation to a Linux target machine.

cdc_rsync* support efficient copy of files by using content-defined
chunking (CDC) to identify chunks within files that can be reused.

asset_stream_manager + cdc_fuse_fs support efficient streaming of a
local directory to a remote virtual file system based on FUSE. It also
employs CDC to identify and reuse unchanged data chunks.
This commit is contained in:
Christian Schneider
2022-10-07 10:47:04 +02:00
commit 4326e972ac
364 changed files with 49410 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

560
common/BUILD Normal file
View File

@@ -0,0 +1,560 @@
package(default_visibility = ["//visibility:public"])
cc_library(
name = "buffer",
srcs = ["buffer.cc"],
hdrs = ["buffer.h"],
)
cc_test(
name = "buffer_test",
srcs = ["buffer_test.cc"],
deps = [
":buffer",
"@com_google_googletest//:gtest",
"@com_google_googletest//:gtest_main",
],
)
cc_library(
name = "clock",
srcs = ["clock.cc"],
hdrs = ["clock.h"],
deps = [
":platform",
"@com_google_absl//absl/strings:str_format",
],
)
cc_library(
name = "dir_iter",
srcs = ["dir_iter.cc"],
hdrs = ["dir_iter.h"],
copts = select({
"//tools:windows": [
# Additional warnings from @com_github_dirent
"/wd4505", # unreferenced function with internal linkage has been removed
],
"//conditions:default": ["/wd4505"],
}),
deps = [
":path",
":platform",
"@com_github_dirent//:dirent",
"@com_google_absl//absl/status",
"@com_google_absl//absl/strings",
"@com_google_absl//absl/strings:str_format",
],
)
cc_test(
name = "dir_iter_test",
srcs = ["dir_iter_test.cc"],
data = ["testdata/root.txt"] + glob(["testdata/dir_iter/**"]),
deps = [
":dir_iter",
":status_test_macros",
":test_main",
"@com_google_googletest//:gtest",
],
)
cc_library(
name = "errno_mapping",
srcs = ["errno_mapping.cc"],
hdrs = ["errno_mapping.h"],
deps = [
"@com_google_absl//absl/status",
"@com_google_absl//absl/strings",
],
)
cc_test(
name = "errno_mapping_test",
srcs = ["errno_mapping_test.cc"],
deps = [
":errno_mapping",
":status_test_macros",
"@com_google_googletest//:gtest",
"@com_google_googletest//:gtest_main",
],
)
cc_library(
name = "file_watcher",
srcs = [
"file_watcher_win.cc",
],
hdrs = [
"file_watcher_win.h",
],
# Required for ReadDirectoryChangesExW (requires Win10 1709).
copts = ["/D_WIN32_WINNT=0x0A00"],
target_compatible_with = ["@platforms//os:windows"],
deps = [
":log",
":path",
":platform",
":scoped_handle",
":status",
":stopwatch",
":util",
"@com_google_absl//absl/status",
"@com_google_absl//absl/status:statusor",
"@com_google_absl//absl/strings:str_format",
"@com_google_absl//absl/synchronization",
],
)
cc_test(
name = "file_watcher_test",
srcs = ["file_watcher_win_test.cc"],
target_compatible_with = ["@platforms//os:windows"],
deps = [
":file_watcher",
":path",
":platform",
":status_test_macros",
":util",
"@com_google_googletest//:gtest",
"@com_google_googletest//:gtest_main",
],
)
cc_library(
name = "grpc_status",
hdrs = ["grpc_status.h"],
deps = [
"@com_github_grpc_grpc//:grpc++",
"@com_google_absl//absl/status",
],
)
cc_library(
name = "log",
srcs = ["log.cc"],
hdrs = ["log.h"],
deps = [
":clock",
":platform",
"@com_google_absl//absl/strings:str_format",
"@com_google_absl//absl/synchronization",
],
)
cc_test(
name = "log_test",
srcs = ["log_test.cc"],
deps = [
":log",
":path",
":status_test_macros",
"@com_google_absl//absl/strings",
"@com_google_googletest//:gtest",
"@com_google_googletest//:gtest_main",
],
)
cc_library(
name = "path",
srcs = ["path.cc"],
hdrs = ["path.h"],
linkopts = select({
"//tools:windows": [
"/DEFAULTLIB:ole32.lib", # CoTaskMemFree
"/DEFAULTLIB:shell32.lib", # SHGetKnownFolderPath
],
"//conditions:default": [],
}),
deps = [
":buffer",
":errno_mapping",
":log",
":platform",
":status",
":status_macros",
":util",
"@com_google_absl//absl/status",
"@com_google_absl//absl/status:statusor",
"@com_google_absl//absl/strings",
"@com_google_absl//absl/strings:str_format",
],
)
cc_test(
name = "path_test",
srcs = ["path_test.cc"],
env_inherit = select({
"//tools:windows": [
"ProgramFiles(x86)",
"USERPROFILE",
],
"//conditions:default": [],
}),
deps = [
":path",
":status_test_macros",
":test_main",
":util",
"@com_google_googletest//:gtest",
],
)
cc_library(
name = "path_filter",
srcs = ["path_filter.cc"],
hdrs = ["path_filter.h"],
)
cc_test(
name = "path_filter_test",
srcs = ["path_filter_test.cc"],
deps = [
":path_filter",
":test_main",
"@com_google_googletest//:gtest",
],
)
cc_library(
name = "platform",
hdrs = ["platform.h"],
)
cc_library(
name = "port_manager",
srcs = ["port_manager_win.cc"],
hdrs = ["port_manager.h"],
target_compatible_with = ["@platforms//os:windows"],
deps = [
":remote_util",
":status",
":stopwatch",
":util",
"@com_google_absl//absl/status:statusor",
],
)
cc_test(
name = "port_manager_test",
srcs = ["port_manager_test.cc"],
target_compatible_with = ["@platforms//os:windows"],
deps = [
":port_manager",
":status_test_macros",
":stub_process",
":test_main",
":testing_clock",
"@com_google_googletest//:gtest",
],
)
cc_library(
name = "process",
srcs = ["process_win.cc"],
hdrs = ["process.h"],
target_compatible_with = ["@platforms//os:windows"],
deps = [
":log",
":scoped_handle",
":status",
":status_test_macros",
":stopwatch",
":util",
"@com_google_absl//absl/status",
"@com_google_absl//absl/status:statusor",
"@com_google_absl//absl/strings:str_format",
],
)
cc_test(
name = "process_test",
srcs = ["process_test.cc"],
target_compatible_with = ["@platforms//os:windows"],
deps = [
":process",
":test_main",
"@com_google_googletest//:gtest",
],
)
cc_library(
name = "status_macros",
hdrs = ["status_macros.h"],
deps = [
":status",
"@com_google_absl//absl/status",
"@com_google_absl//absl/status:statusor",
"@com_google_absl//absl/strings:str_format",
],
)
cc_library(
name = "status_test_macros",
hdrs = ["status_test_macros.h"],
deps = [
":platform",
"@com_google_absl//absl/status",
"@com_google_absl//absl/status:statusor",
],
)
cc_library(
name = "stub_process",
srcs = ["stub_process.cc"],
hdrs = ["stub_process.h"],
deps = [
":process",
":status_macros",
],
)
cc_library(
name = "remote_util",
srcs = ["remote_util.cc"],
hdrs = ["remote_util.h"],
deps = [
":platform",
":process",
":sdk_util",
":util",
"@com_google_absl//absl/status",
],
)
cc_test(
name = "remote_util_test",
srcs = ["remote_util_test.cc"],
deps = [
":remote_util",
":test_main",
],
)
cc_library(
name = "semaphore",
srcs = ["semaphore.cc"],
hdrs = ["semaphore.h"],
)
cc_test(
name = "semaphore_test",
srcs = ["semaphore_test.cc"],
deps = [
":semaphore",
":test_main",
"@com_google_googletest//:gtest",
],
)
cc_library(
name = "gamelet_component",
srcs = ["gamelet_component.cc"],
hdrs = ["gamelet_component.h"],
deps = [
":path",
":platform",
":status",
"@com_google_absl//absl/status",
"@com_google_absl//absl/strings:str_format",
],
)
cc_test(
name = "gamelet_component_test",
srcs = ["gamelet_component_test.cc"],
data = ["testdata/root.txt"] + glob(["testdata/gamelet_component/**"]),
deps = [
":gamelet_component",
":status_test_macros",
":test_main",
"@com_google_googletest//:gtest",
],
)
cc_library(
name = "stats_collector",
srcs = ["stats_collector.cc"],
hdrs = ["stats_collector.h"],
deps = [
":log",
":stopwatch",
"@com_google_absl//absl/strings:str_format",
"@com_google_absl//absl/synchronization",
],
)
cc_library(
name = "status",
srcs = ["status.cc"],
hdrs = ["status.h"],
deps = [
":platform",
"@com_google_absl//absl/status",
"@com_google_absl//absl/strings",
"@com_google_absl//absl/strings:str_format",
],
)
cc_library(
name = "stopwatch",
srcs = ["stopwatch.cc"],
hdrs = ["stopwatch.h"],
deps = [
":clock",
"@com_google_absl//absl/time",
],
)
cc_test(
name = "stopwatch_test",
srcs = ["stopwatch_test.cc"],
deps = [
":stopwatch",
":test_main",
":testing_clock",
"@com_google_googletest//:gtest",
],
)
cc_library(
name = "sdk_util",
srcs = ["sdk_util.cc"],
hdrs = ["sdk_util.h"],
target_compatible_with = ["@platforms//os:windows"],
deps = [
":path",
":platform",
":status_macros",
"@com_google_absl//absl/status",
"@com_google_absl//absl/status:statusor",
],
)
cc_test(
name = "sdk_util_test",
srcs = ["sdk_util_test.cc"],
deps = [
":sdk_util",
":status_test_macros",
":test_main",
"@com_google_googletest//:gtest",
],
)
cc_library(
name = "threadpool",
srcs = ["threadpool.cc"],
hdrs = ["threadpool.h"],
deps = ["@com_google_absl//absl/synchronization"],
)
cc_test(
name = "threadpool_test",
srcs = ["threadpool_test.cc"],
deps = [
":semaphore",
":test_main",
":threadpool",
"@com_google_googletest//:gtest",
],
)
cc_library(
name = "testing_clock",
srcs = ["testing_clock.cc"],
hdrs = ["testing_clock.h"],
deps = [
":clock",
":stopwatch",
],
)
cc_library(
name = "test_main",
srcs = ["test_main.cc"],
hdrs = ["test_main.h"],
deps = [
":path",
"@bazel_tools//tools/cpp/runfiles",
"@com_google_googletest//:gtest",
],
)
cc_library(
name = "thread_safe_map",
hdrs = ["thread_safe_map.h"],
deps = ["@com_google_absl//absl/synchronization"],
)
cc_test(
name = "thread_safe_map_test",
srcs = ["thread_safe_map_test.cc"],
deps = [
":thread_safe_map",
"@com_google_googletest//:gtest",
"@com_google_googletest//:gtest_main",
],
)
cc_library(
name = "scoped_handle",
srcs = ["scoped_handle_win.cc"],
hdrs = ["scoped_handle_win.h"],
target_compatible_with = ["@platforms//os:windows"],
)
cc_library(
name = "util",
srcs = ["util.cc"],
hdrs = ["util.h"],
deps = [
":platform",
"@com_google_absl//absl/random",
"@com_google_absl//absl/strings",
"@com_google_absl//absl/strings:str_format",
"@com_google_absl//absl/time",
],
)
cc_test(
name = "util_test",
srcs = ["util_test.cc"],
deps = [
":util",
"@com_google_googletest//:gtest",
"@com_google_googletest//:gtest_main",
],
)
cc_library(
name = "url",
srcs = ["url.cc"],
hdrs = ["url.h"],
deps = [
"@com_google_absl//absl/status:statusor",
"@com_google_absl//absl/strings:str_format",
],
)
cc_test(
name = "url_test",
srcs = ["url_test.cc"],
deps = [
":status_test_macros",
":url",
"@com_google_absl//absl/status:statusor",
"@com_google_googletest//:gtest",
"@com_google_googletest//:gtest_main",
],
)
filegroup(
name = "all_test_sources",
srcs = glob(["*_test.cc"]),
)
filegroup(
name = "all_test_data",
srcs = glob(["testdata/**"]),
)

88
common/buffer.cc Normal file
View File

@@ -0,0 +1,88 @@
// Copyright 2022 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "common/buffer.h"
#include <cstring>
#include <utility>
namespace cdc_ft {
Buffer::Buffer() = default;
Buffer::Buffer(size_t size) : size_(size) {
reserve(size);
resize(size);
}
Buffer::Buffer(std::initializer_list<char> list) {
reserve(list.size());
resize(list.size());
memcpy(data_, list.begin(), list.size());
}
Buffer::Buffer(Buffer&& other) noexcept { *this = std::move(other); }
Buffer& Buffer::operator=(Buffer&& other) noexcept {
if (data_) {
free(data_);
}
size_ = other.size_;
capacity_ = other.capacity_;
data_ = other.data_;
other.size_ = 0;
other.capacity_ = 0;
other.data_ = nullptr;
return *this;
}
Buffer::~Buffer() {
if (data_) {
free(data_);
data_ = nullptr;
capacity_ = 0;
}
}
bool Buffer::operator==(const Buffer& other) const {
return size_ == other.size_ && memcmp(data_, other.data_, size_) == 0;
}
bool Buffer::operator!=(const Buffer& other) const { return !(*this == other); }
void Buffer::resize(size_t new_size) {
size_ = new_size;
if (capacity_ < size_) {
capacity_ = size_ + size_ / 2;
data_ = static_cast<char*>(realloc(data_, capacity_));
}
}
void Buffer::reserve(size_t capacity) {
if (capacity_ < capacity) {
capacity_ = capacity;
data_ = static_cast<char*>(realloc(data_, capacity_));
}
}
void Buffer::append(const void* data, size_t data_size) {
size_t prev_size = size_;
resize(prev_size + data_size);
memcpy(data_ + prev_size, data, data_size);
}
} // namespace cdc_ft

87
common/buffer.h Normal file
View File

@@ -0,0 +1,87 @@
/*
* Copyright 2022 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef COMMON_BUFFER_H_
#define COMMON_BUFFER_H_
#include <cstdlib>
#include <initializer_list>
namespace cdc_ft {
// Buffer for generic data blobs. It is a managed and dynamically sized memory
// buffer with an interface similar to that of std::vector. It can be used as an
// in-place replacement in some use cases like reading files.
// The class solves a performance problem on Linux where resize() can be
// relatively expensive since a vector default-constructs elements. This can
// be a big performance bottleneck in some cases.
class Buffer {
public:
Buffer();
explicit Buffer(size_t size);
Buffer(std::initializer_list<char> list);
~Buffer();
// Non-copyable, non-assignable for safety.
Buffer(const Buffer& other) = delete;
Buffer& operator=(const Buffer& other) = delete;
Buffer(Buffer&& other) noexcept;
Buffer& operator=(Buffer&& other) noexcept;
bool operator==(const Buffer& other) const;
bool operator!=(const Buffer& other) const;
// Resizes the buffer. Only reallocates if the size increases. In that case,
// allocates 1.5x more than required. If size shrinks, keeps the buffer.
// Does not initialize the buffer.
void resize(size_t new_size);
// Makes sure the buffer capacity is at least the given number of bytes.
void reserve(size_t capacity);
// Returns the current size of the buffer.
size_t size() const { return size_; }
// Returns true if the size is zero.
bool empty() const { return size_ == 0; }
// Returns the currently allocated buffer size.
size_t capacity() const { return capacity_; }
// Resizes the buffer to 0.
void clear() { size_ = 0; }
// Returns the data pointer.
char* data() { return data_; }
// Returns the data pointer.
const char* data() const { return data_; }
// Appends |data| of size |data_size| to the end of the buffer. The behavior
// is undefined if the provided memory range overlaps with the buffer's data.
void append(const void* data, size_t data_size);
private:
char* data_ = nullptr;
size_t size_ = 0;
size_t capacity_ = 0;
};
} // namespace cdc_ft
#endif // COMMON_BUFFER_H_

167
common/buffer_test.cc Normal file
View File

@@ -0,0 +1,167 @@
// Copyright 2022 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "common/buffer.h"
#include <cstring>
#include "gtest/gtest.h"
namespace cdc_ft {
namespace {
TEST(BufferTest, ConstructDefault) {
Buffer b;
EXPECT_EQ(b.size(), 0);
EXPECT_EQ(b.data(), nullptr);
}
TEST(BufferTest, ConstructWithSize) {
Buffer b(32);
EXPECT_EQ(b.size(), 32);
EXPECT_EQ(b.capacity(), 32);
EXPECT_NE(b.data(), nullptr);
// This shouldn't crash.
memset(b.data(), 0, b.size());
}
TEST(BufferTest, ConstructWithInitializerList) {
Buffer b({'1', '2', '3', '4', '5', '6', '7', '8', '9'});
EXPECT_EQ(b.size(), 9);
EXPECT_EQ(b.capacity(), 9);
EXPECT_NE(b.data(), nullptr);
EXPECT_EQ(memcmp(b.data(), "123456789", 9), 0);
}
TEST(BufferTest, MoveConstructor) {
Buffer b({9});
Buffer b2(std::move(b));
EXPECT_EQ(b.size(), 0);
EXPECT_EQ(b.data(), nullptr);
EXPECT_EQ(b.capacity(), 0);
EXPECT_EQ(b2.size(), 1);
EXPECT_NE(b2.data(), nullptr);
EXPECT_EQ(*b2.data(), 9);
}
TEST(BufferTest, MoveOperator) {
Buffer b({9});
Buffer b2({12, 13});
b2 = std::move(b);
EXPECT_EQ(b.size(), 0);
EXPECT_EQ(b.data(), nullptr);
EXPECT_EQ(b.capacity(), 0);
EXPECT_EQ(b2.size(), 1);
EXPECT_NE(b2.data(), nullptr);
EXPECT_EQ(*b2.data(), 9);
}
TEST(BufferTest, EqualsOperator) {
Buffer b({1});
Buffer b2({1});
Buffer b3({2});
EXPECT_TRUE(b == b2);
EXPECT_FALSE(b == b3);
EXPECT_FALSE(b != b2);
EXPECT_TRUE(b != b3);
}
TEST(BufferTest, SizeIncrease) {
Buffer b(8);
EXPECT_EQ(b.size(), 8);
EXPECT_EQ(b.capacity(), 8);
EXPECT_NE(b.data(), nullptr);
memcpy(b.data(), "01234567", 8);
// Should realloc the buffer, but keep the data.
b.resize(10);
EXPECT_EQ(b.size(), 10);
EXPECT_EQ(b.capacity(), 15);
EXPECT_NE(b.data(), nullptr);
EXPECT_EQ(memcmp(b.data(), "01234567", 8), 0);
memcpy(b.data(), "0123456789", 10);
// Should not realloc the buffer and keep data.
b.resize(12);
EXPECT_EQ(b.size(), 12);
EXPECT_EQ(b.capacity(), 15);
EXPECT_NE(b.data(), nullptr);
EXPECT_EQ(memcmp(b.data(), "0123456789", 10), 0);
}
TEST(BufferTest, SizeDecrease) {
Buffer b(8);
EXPECT_EQ(b.size(), 8);
EXPECT_EQ(b.capacity(), 8);
EXPECT_NE(b.data(), nullptr);
memcpy(b.data(), "01234567", 8);
// Should not realloc the buffer and keep data.
b.resize(2);
EXPECT_EQ(b.size(), 2);
EXPECT_EQ(b.capacity(), 8);
EXPECT_NE(b.data(), nullptr);
EXPECT_EQ(memcmp(b.data(), "01", 2), 0);
}
TEST(BufferTest, Reserve) {
Buffer b({1});
Buffer b2({1});
Buffer b3({2});
EXPECT_TRUE(b == b2);
EXPECT_FALSE(b == b3);
EXPECT_FALSE(b != b2);
EXPECT_TRUE(b != b3);
}
TEST(BufferTest, Empty) {
Buffer b;
EXPECT_TRUE(b.empty());
b = {};
EXPECT_TRUE(b.empty());
b = {1};
EXPECT_FALSE(b.empty());
}
TEST(BufferTest, Append) {
Buffer b;
b.append("9", 1);
EXPECT_EQ(b, Buffer({'9'}));
b.append("123", 3);
EXPECT_EQ(b, Buffer({'9', '1', '2', '3'}));
}
TEST(BufferTest, Clear) {
Buffer b(8);
EXPECT_EQ(b.size(), 8);
EXPECT_EQ(b.capacity(), 8);
EXPECT_NE(b.data(), nullptr);
b.clear();
EXPECT_EQ(b.size(), 0);
EXPECT_EQ(b.capacity(), 8);
EXPECT_NE(b.data(), nullptr);
}
} // namespace
} // namespace cdc_ft

64
common/clock.cc Normal file
View File

@@ -0,0 +1,64 @@
// Copyright 2022 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "common/clock.h"
#include "absl/strings/str_format.h"
#include "common/platform.h"
namespace cdc_ft {
// static
DefaultSteadyClock* DefaultSteadyClock::GetInstance() {
static DefaultSteadyClock instance;
return &instance;
}
SteadyClock::Timestamp DefaultSteadyClock::Now() const {
return std::chrono::steady_clock::now();
}
std::string SystemClock::FormatNow(const char* format,
bool append_millis) const {
const Timestamp now = Now();
const std::time_t now_t = std::chrono::system_clock::to_time_t(now);
tm now_tm;
#if PLATFORM_WINDOWS
localtime_s(&now_tm, &now_t);
#elif PLATFORM_LINUX
localtime_r(&now_t, &now_tm);
#endif
std::stringstream ss;
ss << std::put_time(&now_tm, format);
if (append_millis) {
const int millis = std::chrono::duration_cast<std::chrono::milliseconds>(
now - std::chrono::system_clock::from_time_t(now_t))
.count();
ss << std::setfill('0') << std::setw(3) << millis;
}
return ss.str();
}
// static
DefaultSystemClock* DefaultSystemClock::GetInstance() {
static DefaultSystemClock instance;
return &instance;
}
SystemClock::Timestamp DefaultSystemClock::Now() const {
return std::chrono::system_clock::now();
}
} // namespace cdc_ft

61
common/clock.h Normal file
View File

@@ -0,0 +1,61 @@
/*
* Copyright 2022 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef COMMON_CLOCK_H_
#define COMMON_CLOCK_H_
#include <chrono>
#include <string>
namespace cdc_ft {
// Clock that never runs backwards. Useful for stopwatch etc.
class SteadyClock {
public:
using Timestamp = std::chrono::time_point<std::chrono::steady_clock>;
virtual ~SteadyClock() = default;
virtual Timestamp Now() const = 0;
};
class DefaultSteadyClock : public SteadyClock {
public:
static DefaultSteadyClock* GetInstance();
Timestamp Now() const override;
};
// Clock that matches the system's clock.
class SystemClock {
public:
using Timestamp = std::chrono::time_point<std::chrono::system_clock>;
virtual ~SystemClock() = default;
virtual Timestamp Now() const = 0;
// Formats the current timestamp. |format| is the format according to the
// std::put_time specification, see
// https://en.cppreference.com/w/cpp/io/manip/put_time.
// If |append_millis| is true, appends the milliseconds formatted as %03i.
std::string FormatNow(const char* format, bool append_millis) const;
};
class DefaultSystemClock : public SystemClock {
public:
static DefaultSystemClock* GetInstance();
Timestamp Now() const override;
};
} // namespace cdc_ft
#endif // COMMON_CLOCK_H_

287
common/dir_iter.cc Normal file
View File

@@ -0,0 +1,287 @@
// Copyright 2022 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "common/dir_iter.h"
#include <errno.h>
#include <string.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <cassert>
#include <list>
#if defined(_MSC_VER)
#include "dirent.h"
#else
#include <dirent.h>
#endif
#include <algorithm>
#include <memory>
#include "absl/strings/str_cat.h"
#include "absl/strings/str_format.h"
#include "common/errno_mapping.h"
#include "common/path.h"
namespace cdc_ft {
namespace {
const std::string empty_string;
}
// Allows testing if a specific flag is set using the & operator.
inline int operator&(DirectorySearchFlags a, DirectorySearchFlags b) {
using T = std::underlying_type_t<DirectorySearchFlags>;
return static_cast<int>(static_cast<T>(a) & static_cast<T>(b));
}
class DirectoryEntry::Impl {
friend class DirectoryEntry;
public:
Impl(const std::string& rel_path, const dirent* dent)
: rel_path(rel_path),
name(dent ? dent->d_name : ""),
type(dent ? dent->d_type : 0) {}
const std::string rel_path;
const std::string name;
const int type;
};
DirectoryEntry::DirectoryEntry() : impl_(nullptr) {}
DirectoryEntry::~DirectoryEntry() {
if (impl_) delete impl_;
}
bool DirectoryEntry::Valid() const { return impl_ && impl_->type; }
bool DirectoryEntry::IsDir() const { return impl_ && impl_->type & DT_DIR; }
bool DirectoryEntry::IsRegularFile() const {
return impl_ && impl_->type & DT_REG;
}
bool DirectoryEntry::IsSymlink() const { return impl_ && impl_->type & DT_LNK; }
const std::string& DirectoryEntry::Name() const {
return impl_ ? impl_->name : empty_string;
}
const std::string& DirectoryEntry::RelPath() const {
return impl_ ? impl_->rel_path : empty_string;
}
std::string DirectoryEntry::RelPathName() const {
return impl_ ? path::Join(impl_->rel_path, impl_->name) : std::string();
}
void DirectoryEntry::Clear() { SetImpl(nullptr); }
void DirectoryEntry::SetImpl(Impl* impl) {
if (impl_) delete impl_;
impl_ = impl;
}
class DirectoryIterator::Impl {
public:
Impl(const std::string& path, DirectorySearchFlags results, bool recursive)
: path_(path),
flags_(results),
recursive_(recursive),
last_dir_(nullptr) {}
~Impl() {
while (!DirsEmpty()) PopDir();
}
inline const std::string& Path() const { return path_; }
inline bool ReturnDirs() const {
return flags_ & DirectorySearchFlags::kDirectories;
}
inline bool ReturnFiles() const {
return flags_ & DirectorySearchFlags::kFiles;
}
inline bool Recursive() const { return recursive_; }
// Reads the next directory entry from the topmost opened directory. Returns
// nullptr when all directory entries have been read or in case of an error.
// Check errno to distinguish between those two cases.
inline struct dirent* NextDirEntry() {
assert(!dirs_.empty());
// Update relative path if it changed.
DIR* dir = dirs_.back().dir;
if (last_dir_ != dir) {
UpdateRelPath();
last_dir_ = dir;
}
// Reset previous error so that we know if we reached the end the dir.
errno = 0;
return readdir(dir);
}
inline const std::string& RelPath() const { return rel_path_; }
inline std::string DirsPath() { return path::Join(path_, rel_path_); }
inline void PushDir(DIR* dir, const std::string& name) {
assert(dir != nullptr);
dirs_.push_back({dir, name});
}
inline void PopDir() {
assert(!dirs_.empty());
closedir(dirs_.back().dir);
dirs_.pop_back();
}
inline bool DirsEmpty() const { return dirs_.empty(); }
inline absl::Status Status() const { return status_; }
inline void SetStatus(absl::Status status) { status_ = status; }
private:
// On Windows, the struct DIR::ent doesn't seem properly aligned, so the
// DIR::ent::d_name field is unusable. Thus, we store the DIR pointer along
// with its name in this struct.
struct OpenedDir {
DIR* dir;
std::string name;
};
inline void UpdateRelPath() {
rel_path_.clear();
bool first = true;
for (const OpenedDir& dir : dirs_) {
if (first) {
// The first component is already included in path_.
first = false;
continue;
}
path::Append(&rel_path_, dir.name);
}
}
const std::string path_;
const DirectorySearchFlags flags_;
const bool recursive_;
absl::Status status_;
std::list<OpenedDir> dirs_;
const DIR* last_dir_;
std::string rel_path_;
};
DirectoryIterator::DirectoryIterator() : impl_(nullptr) {}
DirectoryIterator::DirectoryIterator(const std::string& path,
DirectorySearchFlags results,
bool recursive)
: impl_(nullptr) {
Open(path, results, recursive);
}
DirectoryIterator::~DirectoryIterator() {
if (impl_) delete impl_;
}
absl::Status DirectoryIterator::Status() const {
return impl_ ? impl_->Status() : absl::OkStatus();
}
bool DirectoryIterator::Valid() const {
return impl_ && impl_->Status().ok() && !impl_->DirsEmpty();
}
const std::string& DirectoryIterator::Path() const {
return impl_ ? impl_->Path() : empty_string;
}
bool DirectoryIterator::Open(const std::string& path,
DirectorySearchFlags results, bool recursive) {
if (impl_) delete impl_;
impl_ = new Impl(path, results, recursive);
DIR* dir = opendir(path.c_str());
if (dir) {
impl_->PushDir(dir, path::BaseName(path));
return true;
}
impl_->SetStatus(ErrnoToCanonicalStatus(
errno, absl::StrFormat("Failed to open directory '%s'", path)));
return false;
}
bool DirectoryIterator::NextEntry(DirectoryEntry* entry) {
if (!impl_ || !impl_->Status().ok()) return false;
// Reset last error to not report a previous one below.
errno = 0;
while (!impl_->DirsEmpty()) {
struct dirent* dent = impl_->NextDirEntry();
if (!dent) {
if (!errno) {
// We have reached the end of this directory.
impl_->PopDir();
continue;
}
break;
}
// Handle directories.
if (dent->d_type & DT_DIR) {
// Skip "." and ".." directory entries.
if (!strcmp(dent->d_name, ".") || !strcmp(dent->d_name, "..")) {
continue;
}
// For recursive traversal, push new directory on top of the stack.
if (impl_->Recursive()) {
std::string dname(dent->d_name);
std::string subdir = path::Join(impl_->DirsPath(), dname);
DIR* dir = opendir(subdir.c_str());
if (dir) {
impl_->PushDir(dir, dname);
} else if (errno == EACCES) {
// Ignore access errors and proceed.
} else {
impl_->SetStatus(ErrnoToCanonicalStatus(
errno, absl::StrFormat("Failed to open directory '%s'", subdir)));
return false;
}
}
if (impl_->ReturnDirs()) {
entry->SetImpl(new DirectoryEntry::Impl(impl_->RelPath(), dent));
return true;
}
} else if (dent->d_type & DT_REG) {
// Handle regular files.
if (impl_->ReturnFiles()) {
entry->SetImpl(new DirectoryEntry::Impl(impl_->RelPath(), dent));
return true;
}
}
}
if (errno) {
impl_->SetStatus(ErrnoToCanonicalStatus(
errno, absl::StrFormat("Failed to iterate over directory '%s'",
impl_->DirsPath())));
}
return false;
}
}; // namespace cdc_ft

145
common/dir_iter.h Normal file
View File

@@ -0,0 +1,145 @@
/*
* Copyright 2022 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef COMMON_DIR_ITER_H_
#define COMMON_DIR_ITER_H_
#include <cstdio>
#include <string>
#include "absl/status/status.h"
#include "absl/strings/string_view.h"
namespace cdc_ft {
// Flags for selecting the type of directory entries when iterating. This allows
// one to narrow down the results that a DirectoryIterator returns.
enum class DirectorySearchFlags {
kNone = 0,
kDirectories = (1 << 0),
kFiles = (1 << 1),
kFilesAndDirectories = kFiles | kDirectories,
};
class DirectoryIterator;
// A DirectoryEntry describes a file or a directory that was found when
// iterating through the file system.
class DirectoryEntry {
public:
DirectoryEntry();
~DirectoryEntry();
// Copy and assignment are disabled due to the separate implementation class.
DirectoryEntry(DirectoryEntry const&) = delete;
DirectoryEntry& operator=(DirectoryEntry const&) = delete;
// Returns true if the entry describes an existing item.
bool Valid() const;
// Returns true if the entry is valid and describes a directory.
bool IsDir() const;
// Returns true if the entry is valid and describes a regular file.
bool IsRegularFile() const;
// Returns true if the entry is valid and describes a symlink.
bool IsSymlink() const;
// Returns the name for this entry, without any path component.
const std::string& Name() const;
// Returns the relative path from the originating DirectoryIterator to the
// directory that contains this entry (excluding this entry).
const std::string& RelPath() const;
// Returns the relative path and filename from the originating
// DirectoryIterator to the this entry (including this entry).
std::string RelPathName() const;
// Frees all data associated with this entry. After calling Clear(), this
// entry is no longer valid.
void Clear();
private:
class Impl;
friend class DirectoryIterator;
void SetImpl(Impl* impl);
Impl* impl_;
};
// This class allows recursive listing of directory contents.
class DirectoryIterator {
public:
// Default constructor.
DirectoryIterator();
// Contructs a new iterator and immediately calls Open() using the given
// parameters.
DirectoryIterator(
const std::string& path,
DirectorySearchFlags results = DirectorySearchFlags::kFilesAndDirectories,
bool recursive = true);
// Destructor
~DirectoryIterator();
// Copy and assignment are disabled due to the separate implementation class.
DirectoryIterator(DirectoryIterator const&) = delete;
DirectoryIterator& operator=(DirectoryIterator const&) = delete;
// Returns any error that might have occured so far. Note that the iterator
// ignores any permission errors that might occur when recursing into
// restricted sub-directories and just continues with the next directory.
absl::Status Status() const;
// Returns true as long as a directory has been opened, no error has occured,
// and a call to NextEntry() has a chance to succeed.
bool Valid() const;
// Returns the path that was given to open the first directory.
const std::string& Path() const;
// Opens the given directory path for reading. If this method returns true, a
// directory entry may be fetched by calling NextEntry(). The results
// parameter can be used to control which types of directory entries a call to
// NextEntry() might yield. The recustive parameter controls whether or not
// the iterator recurses into sub-directories in a DFS manner.
//
// In case of an error (including permission errors), this function returns
// false. Check Status() for the actual error in that case.
bool Open(
const std::string& path,
DirectorySearchFlags results = DirectorySearchFlags::kFilesAndDirectories,
bool recursive = true);
// Yields the next directory entry that matches the DirectorySearchFlags that
// were given in the call to Open(). Returns true if a new entry was found,
// false in case of an error or if no more entries are available. Check
// Status() to distinguish between those two cases.
//
// Note: The iterator ignores permission errors that occur in any
// sub-directory and just continues with the next directory.
bool NextEntry(DirectoryEntry* entry);
private:
class Impl;
Impl* impl_;
};
}; // namespace cdc_ft
#endif // COMMON_DIR_ITER_H_

267
common/dir_iter_test.cc Normal file
View File

@@ -0,0 +1,267 @@
// Copyright 2022 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "common/dir_iter.h"
#include <algorithm>
#include <iterator>
#include "common/path.h"
#include "common/platform.h"
#include "common/status_test_macros.h"
#include "common/test_main.h"
#include "gtest/gtest.h"
namespace cdc_ft {
namespace {
class DirectoryIteratorTest : public ::testing::Test {
public:
using StringSet = std::set<std::string>;
DirectoryIteratorTest() {}
void SetUp() override {
expected_files_.insert("a/aa/aaa1.txt");
expected_files_.insert("a/aa/aaa2.txt");
expected_files_.insert("a/aa1.txt");
expected_files_.insert("a/aa2.txt");
expected_files_.insert("a/ab/aab1.txt");
expected_files_.insert("a/ab/aab2.txt");
expected_files_.insert("b/ba/bba1.txt");
expected_files_.insert("b/ba/bba2.txt");
expected_files_.insert("b/bb/bbb1.txt");
expected_files_.insert("b/bb/bbb2.txt");
expected_files_.insert("c/c1.txt");
expected_files_.insert("c/c2.txt");
expected_files_.insert("d/d1.txt");
expected_files_.insert("d/d2.txt");
expected_files_.insert("root.txt");
expected_dirs_.insert("a");
expected_dirs_.insert("a/aa");
expected_dirs_.insert("a/ab");
expected_dirs_.insert("b");
expected_dirs_.insert("b/ba");
expected_dirs_.insert("b/bb");
expected_dirs_.insert("c");
expected_dirs_.insert("d");
}
static StringSet Union(const StringSet& a, const StringSet& b) {
StringSet out(a);
out.insert(b.begin(), b.end());
return out;
}
static StringSet Filter(
const StringSet& items,
std::function<bool(const std::string& item)> filter_fn) {
StringSet out;
std::copy_if(items.begin(), items.end(), std::inserter(out, out.begin()),
[&](const std::string& f) { return !filter_fn(f); });
return out;
}
protected:
std::set<std::string> expected_files_;
std::set<std::string> expected_dirs_;
std::string test_data_dir_ = GetTestDataDir("dir_iter");
};
TEST_F(DirectoryIteratorTest, EmptyDirIterator) {
DirectoryIterator dit;
DirectoryEntry dent;
EXPECT_OK(dit.Status());
EXPECT_FALSE(dit.Valid());
EXPECT_EQ(dit.Path(), std::string());
EXPECT_FALSE(dit.NextEntry(&dent));
}
TEST_F(DirectoryIteratorTest, ValidDirIterator) {
DirectoryIterator dit(test_data_dir_);
DirectoryEntry dent;
EXPECT_OK(dit.Status());
EXPECT_TRUE(dit.Valid());
EXPECT_EQ(dit.Path(), test_data_dir_);
EXPECT_TRUE(dit.NextEntry(&dent));
}
TEST_F(DirectoryIteratorTest, FindFiles) {
DirectoryIterator dit(test_data_dir_, DirectorySearchFlags::kFiles, false);
DirectoryEntry dent;
std::set<std::string> found;
while (dit.NextEntry(&dent)) {
found.insert(path::ToUnix(dent.RelPathName()));
}
EXPECT_OK(dit.Status());
std::set<std::string> expected = Filter(
expected_files_,
[](const std::string& f) { return f.find('/') != std::string::npos; });
EXPECT_EQ(found, expected);
}
TEST_F(DirectoryIteratorTest, FindDirectories) {
DirectoryIterator dit(test_data_dir_, DirectorySearchFlags::kDirectories,
false);
DirectoryEntry dent;
std::set<std::string> found;
while (dit.NextEntry(&dent)) {
found.insert(path::ToUnix(dent.RelPathName()));
}
EXPECT_OK(dit.Status());
std::set<std::string> expected = Filter(
expected_dirs_,
[](const std::string& f) { return f.find('/') != std::string::npos; });
EXPECT_EQ(found, expected);
}
TEST_F(DirectoryIteratorTest, FindFilesAndDirectories) {
DirectoryIterator dit(test_data_dir_,
DirectorySearchFlags::kFilesAndDirectories, false);
DirectoryEntry dent;
std::set<std::string> found;
while (dit.NextEntry(&dent)) {
found.insert(path::ToUnix(dent.RelPathName()));
}
EXPECT_OK(dit.Status());
std::set<std::string> expected = Filter(
Union(expected_dirs_, expected_files_),
[](const std::string& f) { return f.find('/') != std::string::npos; });
EXPECT_EQ(found, expected);
}
TEST_F(DirectoryIteratorTest, FindFilesRecursive) {
DirectoryIterator dit(test_data_dir_, DirectorySearchFlags::kFiles);
DirectoryEntry dent;
std::set<std::string> found;
while (dit.NextEntry(&dent)) {
found.insert(path::ToUnix(dent.RelPathName()));
}
EXPECT_OK(dit.Status());
EXPECT_EQ(found, expected_files_);
}
TEST_F(DirectoryIteratorTest, FindDirectoriesRecursive) {
DirectoryIterator dit(test_data_dir_, DirectorySearchFlags::kDirectories);
DirectoryEntry dent;
std::set<std::string> found;
while (dit.NextEntry(&dent)) {
found.insert(path::ToUnix(dent.RelPathName()));
}
EXPECT_OK(dit.Status());
EXPECT_EQ(found, expected_dirs_);
}
TEST_F(DirectoryIteratorTest, FindFilesAndDirectoriesRecursive) {
DirectoryIterator dit(test_data_dir_);
DirectoryEntry dent;
std::set<std::string> found;
while (dit.NextEntry(&dent)) {
found.insert(path::ToUnix(dent.RelPathName()));
}
EXPECT_OK(dit.Status());
EXPECT_EQ(found, Union(expected_dirs_, expected_files_));
}
TEST_F(DirectoryIteratorTest, DirNotFound) {
DirectoryIterator dit("does/not/exist");
DirectoryEntry dent;
std::set<std::string> found;
while (dit.NextEntry(&dent)) {
found.insert(path::ToUnix(dent.RelPathName()));
}
EXPECT_TRUE(found.empty());
EXPECT_TRUE(absl::IsNotFound(dit.Status()));
}
TEST_F(DirectoryIteratorTest, HandleError) {
// Trying to open a file instead of a directory.
DirectoryIterator dit(path::Join(test_data_dir_, "root.txt"));
DirectoryEntry dent;
std::set<std::string> found;
while (dit.NextEntry(&dent)) {
found.insert(path::ToUnix(dent.RelPathName()));
}
EXPECT_TRUE(found.empty());
EXPECT_NOT_OK(dit.Status());
}
TEST_F(DirectoryIteratorTest, EmptyDirEntry) {
DirectoryEntry dent;
EXPECT_FALSE(dent.Valid());
EXPECT_FALSE(dent.IsDir());
EXPECT_FALSE(dent.IsRegularFile());
EXPECT_FALSE(dent.IsSymlink());
EXPECT_EQ(dent.Name(), std::string());
EXPECT_EQ(dent.RelPath(), std::string());
EXPECT_EQ(dent.RelPathName(), std::string());
}
TEST_F(DirectoryIteratorTest, DirEntryForFile) {
DirectoryIterator dit(test_data_dir_, DirectorySearchFlags::kFiles, false);
DirectoryEntry dent;
EXPECT_TRUE(dit.NextEntry(&dent));
EXPECT_OK(dit.Status());
EXPECT_TRUE(dent.Valid());
EXPECT_FALSE(dent.IsDir());
EXPECT_TRUE(dent.IsRegularFile());
#ifdef PLATFORM_WINDOWS
// On Windows, Bazel does not copy nor link data dependencies.
EXPECT_FALSE(dent.IsSymlink());
#else
// On Linux, Bazel creates symlinks for files in data dependencies.
EXPECT_TRUE(dent.IsSymlink());
#endif
EXPECT_EQ(dent.Name(), "root.txt");
EXPECT_EQ(dent.RelPath(), std::string());
EXPECT_EQ(dent.RelPathName(), "root.txt");
}
TEST_F(DirectoryIteratorTest, DirEntryForDir) {
std::set<std::string> expected_any = Filter(
expected_dirs_,
[](const std::string& f) { return f.find('/') != std::string::npos; });
DirectoryIterator dit(test_data_dir_, DirectorySearchFlags::kDirectories,
false);
DirectoryEntry dent;
EXPECT_TRUE(dit.NextEntry(&dent));
EXPECT_OK(dit.Status());
EXPECT_TRUE(dent.Valid());
EXPECT_TRUE(dent.IsDir());
EXPECT_FALSE(dent.IsRegularFile());
EXPECT_FALSE(dent.IsSymlink());
EXPECT_TRUE(expected_any.find(dent.Name()) != expected_any.end());
EXPECT_EQ(dent.RelPath(), std::string());
EXPECT_TRUE(expected_any.find(dent.RelPathName()) != expected_any.end());
}
TEST_F(DirectoryIteratorTest, DirEntryClear) {
DirectoryIterator dit(test_data_dir_, DirectorySearchFlags::kFiles, false);
DirectoryEntry dent;
EXPECT_TRUE(dit.NextEntry(&dent));
EXPECT_OK(dit.Status());
dent.Clear();
EXPECT_FALSE(dent.Valid());
EXPECT_FALSE(dent.IsDir());
EXPECT_FALSE(dent.IsRegularFile());
EXPECT_FALSE(dent.IsSymlink());
EXPECT_EQ(dent.Name(), std::string());
EXPECT_EQ(dent.RelPath(), std::string());
EXPECT_EQ(dent.RelPathName(), std::string());
}
} // namespace
} // namespace cdc_ft

169
common/errno_mapping.cc Normal file
View File

@@ -0,0 +1,169 @@
// Copyright 2022 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "common/errno_mapping.h"
#include <errno.h>
#include <string>
namespace cdc_ft {
absl::StatusCode ErrnoToCanonicalCode(int error_number) {
switch (error_number) {
case 0:
return absl::StatusCode::kOk;
case EINVAL: // Invalid argument
case ENAMETOOLONG: // Filename too long
case E2BIG: // Argument list too long
case EDESTADDRREQ: // Destination address required
case EDOM: // Mathematics argument out of domain of function
case EFAULT: // Bad address
case EILSEQ: // Illegal byte sequence
case ENOPROTOOPT: // Protocol not available
case ENOSTR: // Not a STREAM
case ENOTSOCK: // Not a socket
case ENOTTY: // Inappropriate I/O control operation
case EPROTOTYPE: // Protocol wrong type for socket
case ESPIPE: // Invalid seek
return absl::StatusCode::kInvalidArgument;
case ETIMEDOUT: // Connection timed out
case ETIME: // Timer expired
return absl::StatusCode::kDeadlineExceeded;
case ENODEV: // No such device
case ENOENT: // No such file or directory
#ifdef ENOMEDIUM
case ENOMEDIUM: // No medium found
#endif
case ENXIO: // No such device or address
case ESRCH: // No such process
return absl::StatusCode::kNotFound;
case EEXIST: // File exists
case EADDRNOTAVAIL: // Address not available
case EALREADY: // Connection already in progress
#ifdef ENOTUNIQ
case ENOTUNIQ: // Name not unique on network
#endif
return absl::StatusCode::kAlreadyExists;
case EPERM: // Operation not permitted
case EACCES: // Permission denied
#ifdef ENOKEY
case ENOKEY: // Required key not available
#endif
case EROFS: // Read only file system
return absl::StatusCode::kPermissionDenied;
case ENOTEMPTY: // Directory not empty
case EISDIR: // Is a directory
case ENOTDIR: // Not a directory
case EADDRINUSE: // Address already in use
case EBADF: // Invalid file descriptor
#ifdef EBADFD
case EBADFD: // File descriptor in bad state
#endif
case EBUSY: // Device or resource busy
case ECHILD: // No child processes
case EISCONN: // Socket is connected
#ifdef EISNAM
case EISNAM: // Is a named type file
#endif
#ifdef ENOTBLK
case ENOTBLK: // Block device required
#endif
case ENOTCONN: // The socket is not connected
case EPIPE: // Broken pipe
#ifdef ESHUTDOWN
case ESHUTDOWN: // Cannot send after transport endpoint shutdown
#endif
case ETXTBSY: // Text file busy
#ifdef EUNATCH
case EUNATCH: // Protocol driver not attached
#endif
return absl::StatusCode::kFailedPrecondition;
case ENOSPC: // No space left on device
#ifdef EDQUOT
case EDQUOT: // Disk quota exceeded
#endif
case EMFILE: // Too many open files
case EMLINK: // Too many links
case ENFILE: // Too many open files in system
case ENOBUFS: // No buffer space available
case ENODATA: // No message is available on the STREAM read queue
case ENOMEM: // Not enough space
case ENOSR: // No STREAM resources
#ifdef EUSERS
case EUSERS: // Too many users
#endif
return absl::StatusCode::kResourceExhausted;
#ifdef ECHRNG
case ECHRNG: // Channel number out of range
#endif
case EFBIG: // File too large
case EOVERFLOW: // Value too large to be stored in data type
case ERANGE: // Result too large
return absl::StatusCode::kOutOfRange;
#ifdef ENOPKG
case ENOPKG: // Package not installed
#endif
case ENOSYS: // Function not implemented
case ENOTSUP: // Operation not supported
case EAFNOSUPPORT: // Address family not supported
#ifdef EPFNOSUPPORT
case EPFNOSUPPORT: // Protocol family not supported
#endif
case EPROTONOSUPPORT: // Protocol not supported
#ifdef ESOCKTNOSUPPORT
case ESOCKTNOSUPPORT: // Socket type not supported
#endif
case EXDEV: // Improper link
return absl::StatusCode::kUnimplemented;
case EAGAIN: // Resource temporarily unavailable
#ifdef ECOMM
case ECOMM: // Communication error on send
#endif
case ECONNREFUSED: // Connection refused
case ECONNABORTED: // Connection aborted
case ECONNRESET: // Connection reset
case EINTR: // Interrupted function call
#ifdef EHOSTDOWN
case EHOSTDOWN: // Host is down
#endif
case EHOSTUNREACH: // Host is unreachable
case ENETDOWN: // Network is down
case ENETRESET: // Connection aborted by network
case ENETUNREACH: // Network unreachable
case ENOLCK: // No locks available
case ENOLINK: // Link has been severed
#ifdef ENONET
case ENONET: // Machine is not on the network
#endif
return absl::StatusCode::kUnavailable;
case EDEADLK: // Resource deadlock avoided
#ifdef ESTALE
case ESTALE: // Stale file handle
#endif
return absl::StatusCode::kAborted;
case ECANCELED: // Operation cancelled
return absl::StatusCode::kCancelled;
default:
return absl::StatusCode::kUnknown;
}
}
absl::Status ErrnoToCanonicalStatus(int error_number,
absl::string_view message) {
return absl::Status(ErrnoToCanonicalCode(error_number),
absl::StrCat(message, ": ", strerror(error_number)));
}
} // namespace cdc_ft

34
common/errno_mapping.h Normal file
View File

@@ -0,0 +1,34 @@
/*
* Copyright 2022 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef COMMON_ERRNO_MAPPING_H_
#define COMMON_ERRNO_MAPPING_H_
#include "absl/status/status.h"
#include "absl/strings/string_view.h"
namespace cdc_ft {
// Converts the errno |error_number| to an absl code.
absl::StatusCode ErrnoToCanonicalCode(int error_number);
// Creates a status by converting the errno |error_number| to an absl code.
absl::Status ErrnoToCanonicalStatus(int error_number,
absl::string_view message);
} // namespace cdc_ft
#endif // COMMON_ERRNO_MAPPING_H_

54
common/errno_mapping_test.cc Normal file
View File

@@ -0,0 +1,54 @@
// Copyright 2022 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "common/errno_mapping.h"
#include <errno.h>
#include <stddef.h>
#include <string>
#include "common/status_test_macros.h"
#include "gtest/gtest.h"
namespace cdc_ft {
namespace {
// Errno value that is hopefully not defined on any OS.
const int kUndefinedErrno = 999123;
TEST(ErrnoMappingTest, ErrnoToCanonicalCode) {
EXPECT_EQ(ErrnoToCanonicalCode(0), absl::StatusCode::kOk);
// Spot-check a few errno values.
EXPECT_EQ(ErrnoToCanonicalCode(EINVAL), absl::StatusCode::kInvalidArgument);
EXPECT_EQ(ErrnoToCanonicalCode(ENOENT), absl::StatusCode::kNotFound);
EXPECT_EQ(ErrnoToCanonicalCode(kUndefinedErrno), absl::StatusCode::kUnknown);
}
TEST(ErrnoMappingTest, ErrnoToCanonicalStatus) {
EXPECT_OK(ErrnoToCanonicalStatus(0, ""));
// Spot-check a few errno values.
EXPECT_ERROR_MSG(InvalidArgument, "test0",
ErrnoToCanonicalStatus(EINVAL, "test0"));
EXPECT_ERROR_MSG(NotFound, "test1", ErrnoToCanonicalStatus(ENOENT, "test1"));
// Apparently errno 999 is known not to be defined.
EXPECT_ERROR_MSG(Unknown, "test2",
ErrnoToCanonicalStatus(kUndefinedErrno, "test2"));
}
} // namespace
} // namespace cdc_ft

629
common/file_watcher_win.cc Normal file
View File

@@ -0,0 +1,629 @@
// Copyright 2022 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "common/file_watcher_win.h"
#define WIN32_LEAN_AND_MEAN
#include <windows.h>
#include <atomic>
#include <thread>
#include "absl/status/statusor.h"
#include "absl/strings/str_format.h"
#include "common/log.h"
#include "common/path.h"
#include "common/scoped_handle_win.h"
#include "common/status.h"
#include "common/status_macros.h"
#include "common/stopwatch.h"
#include "common/util.h"
namespace cdc_ft {
namespace {
static constexpr DWORD kFileNotificationFlags =
FILE_NOTIFY_CHANGE_FILE_NAME | FILE_NOTIFY_CHANGE_DIR_NAME |
FILE_NOTIFY_CHANGE_SIZE | FILE_NOTIFY_CHANGE_LAST_WRITE;
static constexpr size_t kDefaultBufferSize = 1U << 18; // 256 KiB.
void CancelDirIo(ScopedHandle& dir_handle, const std::wstring& dir_path) {
if (dir_handle.IsValid() && !CancelIo(dir_handle.Get())) {
LOG_ERROR("CancelIo() failed for directory '%s': '%s'",
Util::WideToUtf8Str(dir_path), Util::GetLastWin32Error());
}
dir_handle.Close();
}
int64_t ToUnixTime(LARGE_INTEGER windows_time) {
// Jan 1, 1970 (begin of Unix epoch) in 100 ns ticks since Jan 1, 1601 (begin
// of Windows epoch).
constexpr int64_t kUnixEpochOffset = 0x019DB1DED53E8000;
// A Windows tick is 100 ns.
constexpr int64_t kWinTicksPerSecond = 10000000;
return (windows_time.QuadPart - kUnixEpochOffset) / kWinTicksPerSecond;
}
} // namespace
// Background thread to read directory changes.
class AsyncFileWatcher {
public:
enum class FileWatcherState { kDefault, kFailed, kRunning, kShuttingDown };
using FileAction = FileWatcherWin::FileAction;
using FileInfo = FileWatcherWin::FileInfo;
using FileMap = FileWatcherWin::FileMap;
using FilesChangedCb = FileWatcherWin::FilesChangedCb;
using DirRecreatedCb = FileWatcherWin::DirRecreatedCb;
AsyncFileWatcher(std::string dir_path, FilesChangedCb files_changed_cb,
DirRecreatedCb dir_recreated_cb, unsigned int timeout_ms,
bool enforceLegacyReadDirectoryChangesForTesting)
: dir_path_(dir_path),
files_changed_cb_(std::move(files_changed_cb)),
dir_recreated_cb_(std::move(dir_recreated_cb)),
timeout_ms_(timeout_ms) {
// (internal): Check whether ReadDirectoryChangesExW is available.
// It requires Windows 10, version 1709, released October 17, 2017, or
// corresponding Windows Server versions.
if (!enforceLegacyReadDirectoryChangesForTesting) {
read_directory_changes_ex_ =
reinterpret_cast<decltype(ReadDirectoryChangesExW)*>(::GetProcAddress(
::GetModuleHandle(L"Kernel32.dll"), "ReadDirectoryChangesExW"));
}
shutdown_event_ = ScopedHandle(CreateEvent(nullptr, TRUE, FALSE, nullptr));
if (!shutdown_event_.IsValid()) {
SetStatus(absl::InternalError(absl::StrFormat(
"Failed to create shutdown event: '%s'", Util::GetLastWin32Error())));
return;
}
dir_reader_ = std::thread([this]() { WatchDirChanges(); });
}
~AsyncFileWatcher() { Shutdown(); }
absl::Status GetStatus() ABSL_LOCKS_EXCLUDED(status_mutex_) const {
absl::MutexLock mutex(&status_mutex_);
return status_;
}
FileMap GetModifiedFiles() ABSL_LOCKS_EXCLUDED(modified_files_mutex_) {
FileMap files;
absl::MutexLock mutex(&modified_files_mutex_);
std::swap(modified_files_, files);
// Retrieve file stats if ReadDirectoryChangesEx is not available.
if (!read_directory_changes_ex_ && !files.empty()) {
Stopwatch sw;
for (auto& [path, info] : files) {
if (info.action == FileAction::kDeleted) continue;
std::string full_path = path::Join(dir_path_, path);
path::Stats stats;
absl::Status status = path::GetStats(full_path, &stats);
if (!status.ok()) {
LOG_WARNING("Failed to get stats for path '%s'", full_path);
continue;
}
// Don't use the Windows localized timestamp from GetStats.
time_t mtime;
status = path::GetFileTime(full_path, &mtime);
if (!status.ok()) {
LOG_WARNING("Failed to get modification time for path '%s'",
full_path);
continue;
}
info.is_dir = (stats.mode & path::MODE_IFDIR) != 0;
info.size = stats.size;
info.mtime = mtime;
}
LOG_DEBUG("Time to fix file stats: %0.3f sec.", sw.ElapsedSeconds());
}
return files;
}
void ClearModifiedFiles() ABSL_LOCKS_EXCLUDED(modified_files_mutex_) {
absl::MutexLock mutex(&modified_files_mutex_);
modified_files_.clear();
}
uint32_t GetEventCount() ABSL_LOCKS_EXCLUDED(modified_files_mutex_) const {
absl::MutexLock mutex(&modified_files_mutex_);
return event_count_;
}
uint32_t GetDirRecreateEventCount()
ABSL_LOCKS_EXCLUDED(modified_files_mutex_) const {
absl::MutexLock mutex(&modified_files_mutex_);
return dir_recreate_count_;
}
bool IsWatching() ABSL_LOCKS_EXCLUDED(state_mutex) const {
absl::MutexLock mutex(&state_mutex_);
return state_ != FileWatcherState::kDefault &&
state_ != FileWatcherState::kShuttingDown;
}
bool IsShuttingDown() ABSL_LOCKS_EXCLUDED(state_mutex) const {
absl::MutexLock mutex(&state_mutex_);
return state_ == FileWatcherState::kShuttingDown;
}
void Shutdown() {
if (shutdown_event_.IsValid() && !SetEvent(shutdown_event_.Get())) {
LOG_ERROR("SetEvent() for shutdown failed: '%s'",
Util::GetLastWin32Error());
exit(1);
}
if (dir_reader_.joinable()) {
dir_reader_.join();
}
}
private:
// The most important method, which acquires directory handle and reads
// directory changes. It detects removal, creation, and re-creation of the
// watched directory. It is robust to the directory change on any level.
void WatchDirChanges() {
// TODO: Adjust also if there was no directory at the beginning. Currently,
// the directory exists; otherwise, ManifestUpdater would fail.
bool prev_dir_exists = true;
while (true) {
ScopedHandle read_event(CreateEvent(nullptr, /* no security attributes */
TRUE, /* manual-reset event */
FALSE, /* unsignaled */
nullptr)); /* unnamed event object */
if (!read_event.IsValid()) {
SetStatus(absl::InternalError(absl::StrFormat(
"Failed to create read event: '%s'", Util::GetLastWin32Error())));
return;
}
FILE_BASIC_INFO dir_info;
absl::StatusOr<ScopedHandle> status = GetValidDirHandle(&dir_info);
if (!status.ok()) {
SetStatus(status.status());
} else {
// The watched directory exists and its handle is valid.
if (!prev_dir_exists) {
++dir_recreate_count_;
prev_dir_exists = true;
SetStatus(absl::OkStatus());
if (dir_recreated_cb_) dir_recreated_cb_();
}
ReadDirChanges(*status, dir_info, read_event);
if (IsShuttingDown()) {
LOG_DEBUG("Shutting down watching '%s'.", dir_path_);
return;
}
LOG_WARNING("Watched directory '%s' was possibly removed.", dir_path_);
++dir_recreate_count_;
ClearModifiedFiles();
if (dir_recreated_cb_) dir_recreated_cb_();
}
prev_dir_exists = false;
// The shutdown event should be caught on both levels: when the
// watched directory was not removed and when it was recreated. Here
// the shutdown event is considered when the watched directory itself was
// removed or created. If a shutdown was triggered, stop watching. The
// current file-watcher status should not be changed.
if (IsShuttingDown() ||
WaitForSingleObject(shutdown_event_.Get(), timeout_ms_) ==
WAIT_OBJECT_0) {
LOG_DEBUG("Shutting down watching '%s'.", dir_path_);
ResetEvent(shutdown_event_.Get());
return;
}
}
}
// Creates a file handle for the watched directory and requests its
// properties. Returns a directory handle or an error if the directory does
// not exist or its properties could not be retrieved. |dir_info| is an output
// parameter, this is the basic information about the returned directory
// handle for the watched directory. It descibes for example creation and
// modification timestamps.
absl::StatusOr<ScopedHandle> GetValidDirHandle(
FILE_BASIC_INFO* dir_info) const {
assert(dir_info);
// Create handle for the watched directory.
ScopedHandle dir_handle(CreateFileW(
Util::Utf8ToWideStr(dir_path_).c_str(), /* watched directory */
FILE_LIST_DIRECTORY, /* rights to list the directory */
FILE_SHARE_READ | FILE_SHARE_WRITE |
FILE_SHARE_DELETE, /* sharing mode */
nullptr, /* security attributes */
OPEN_EXISTING, /* opens a file iff exists */
FILE_FLAG_BACKUP_SEMANTICS |
FILE_FLAG_OVERLAPPED, /* asynchronous I/O */
nullptr)); /* no template file */
// CreateFileW() failed and returned an invalid handle.
if (!dir_handle.IsValid()) {
return absl::FailedPreconditionError(
absl::StrFormat("Could not start watching '%s': '%s'", dir_path_,
Util::GetLastWin32Error()));
}
// Failed to retrieve basic information about the watched directory.
if (!GetFileInformationByHandleEx(dir_handle.Get(), FileBasicInfo, dir_info,
sizeof(*dir_info))) {
return absl::FailedPreconditionError(absl::StrFormat(
"Could not get information about directory '%s': '%s'", dir_path_,
Util::GetLastWin32Error()));
}
return dir_handle;
}
// Reads changes in the watched directory itself.
// In case of shutdown, the status is ok.
// In case of any error, an error status is set.
// It waits for changes in the watched directory, collects those events in
// |modified_files_|, and notifies the caller about those changes.
void ReadDirChanges(ScopedHandle& dir_handle, FILE_BASIC_INFO& dir_info,
ScopedHandle& read_event) {
OVERLAPPED overlapped;
ZeroMemory(&overlapped, sizeof(overlapped));
// Use FILE_NOTIFY_EXTENDED_INFORMATION if |read_directory_changes_ex_| is
// available and FILE_NOTIFY_INFORMATION otherwise.
constexpr size_t alignment =
std::max<size_t>(alignof(FILE_NOTIFY_EXTENDED_INFORMATION),
alignof(FILE_NOTIFY_INFORMATION));
std::aligned_storage_t<kDefaultBufferSize, alignment> buffer;
overlapped.hEvent = read_event.Get();
BOOL res;
if (read_directory_changes_ex_) {
res = read_directory_changes_ex_(
dir_handle.Get(), &buffer, sizeof(buffer),
true /* check subfolders */, kFileNotificationFlags,
nullptr /* not needed as async read */, &overlapped,
nullptr /* no completion routine */,
ReadDirectoryNotifyExtendedInformation);
} else {
res = ReadDirectoryChangesW(
dir_handle.Get(), &buffer, sizeof(buffer),
true /* check subfolders */, kFileNotificationFlags,
nullptr /* not needed as async read */, &overlapped,
nullptr /* no completion routine */);
}
if (res == FALSE && GetLastError() != ERROR_IO_PENDING) {
SetStatus(absl::InternalError(absl::StrFormat(
"Could not read changes in the watched directory '%s': '%s'",
dir_path_, Util::GetLastWin32Error())));
MaybeSetState(FileWatcherState::kFailed);
CancelDirIo(dir_handle, Util::Utf8ToWideStr(dir_path_));
return;
}
MaybeSetState(FileWatcherState::kRunning);
// Initialize handles to watch: changes in |dir_path_| and shutdown
// events.
HANDLE watch_handles[] = {overlapped.hEvent, shutdown_event_.Get()};
size_t read_index = 0;
size_t shutdown_index = 1;
while (true) {
std::aligned_storage_t<kDefaultBufferSize, alignment> moved_buffer;
const uint32_t wait_index =
WaitForMultipleObjects(static_cast<DWORD>(std::size(watch_handles)),
watch_handles, false, timeout_ms_);
if (wait_index == WAIT_TIMEOUT) {
ScopedHandle dir_handle2(CreateFileW(
Util::Utf8ToWideStr(dir_path_).c_str(), /* watched directory */
FILE_LIST_DIRECTORY, /* rights to list the directory */
FILE_SHARE_READ | FILE_SHARE_WRITE |
FILE_SHARE_DELETE, /* sharing mode */
nullptr, /* security attributes */
OPEN_EXISTING, /* opens a file iff exists */
FILE_FLAG_BACKUP_SEMANTICS |
FILE_FLAG_OVERLAPPED, /* asynchronous I/O */
nullptr)); /* no template file */
FILE_BASIC_INFO dir_info2;
// The directory was removed. The new one was not created.
if (!dir_handle.IsValid() || !dir_handle2.IsValid() ||
!GetFileInformationByHandleEx(dir_handle2.Get(), FileBasicInfo,
&dir_info2, sizeof(dir_info2))) {
SetStatus(absl::FailedPreconditionError(
absl::StrFormat("The watched directory was removed '%s': '%s'",
dir_path_, Util::GetLastWin32Error())));
MaybeSetState(FileWatcherState::kFailed);
break;
}
if (dir_info.CreationTime.QuadPart != dir_info2.CreationTime.QuadPart) {
SetStatus(absl::FailedPreconditionError(absl::StrFormat(
"The watched directory was recreated '%s'", dir_path_)));
MaybeSetState(FileWatcherState::kFailed);
break;
}
// Wait for a new event.
continue;
}
const uint32_t handle_index = wait_index - WAIT_OBJECT_0;
if (handle_index >= std::size(watch_handles)) {
SetStatus(absl::InvalidArgumentError(absl::StrFormat(
"WaitForMultipleObjects failed with invalid handle index %u",
handle_index)));
MaybeSetState(FileWatcherState::kFailed);
break;
}
if (handle_index == shutdown_index) {
ResetEvent(shutdown_event_.Get());
SetStatus(absl::OkStatus());
MaybeSetState(FileWatcherState::kShuttingDown);
break;
}
if (handle_index == read_index) {
DWORD read_bytes = 0;
if (!GetOverlappedResult(dir_handle.Get(), &overlapped, &read_bytes,
TRUE)) {
ResetEvent(overlapped.hEvent);
SetStatus(absl::UnavailableError(
absl::StrFormat("GetOverlappedResult() failed: '%s'",
Util::GetLastWin32Error())));
MaybeSetState(FileWatcherState::kFailed);
break;
}
if (read_bytes == 0) {
SetStatus(absl::DataLossError(absl::StrFormat(
"Buffer overflow: no events were read for the directory '%s'",
dir_path_)));
MaybeSetState(FileWatcherState::kFailed);
break;
}
std::swap(moved_buffer, buffer);
ResetEvent(overlapped.hEvent);
if (read_directory_changes_ex_) {
res = read_directory_changes_ex_(
dir_handle.Get(), &buffer, sizeof(buffer),
true /* check subfolders */, kFileNotificationFlags,
nullptr /* not needed as async read */, &overlapped,
nullptr /* no completion routine */,
ReadDirectoryNotifyExtendedInformation);
ProcessDirChanges<FILE_NOTIFY_EXTENDED_INFORMATION>(&moved_buffer,
read_bytes);
} else {
res = ReadDirectoryChangesW(
dir_handle.Get(), &buffer, sizeof(buffer),
true /* check subfolders */, kFileNotificationFlags,
nullptr /* not needed as async read */, &overlapped,
nullptr /* no completion routine */);
ProcessDirChanges<FILE_NOTIFY_INFORMATION>(&moved_buffer, read_bytes);
}
if (res == FALSE && GetLastError() != ERROR_IO_PENDING) {
SetStatus(absl::FailedPreconditionError(absl::StrFormat(
"Could not read changes in the watched directory '%s': '%s'",
dir_path_, Util::GetLastWin32Error())));
MaybeSetState(FileWatcherState::kFailed);
break;
}
}
}
CancelDirIo(dir_handle, Util::Utf8ToWideStr(dir_path_));
return;
}
// Classifies the changes in the watched directory collected till now,
// deduplicates the changes if needed, and executes the caller's callback.
// BufferType can be FILE_NOTIFY_EXTENDED_INFORMATION or
// FILE_NOTIFY_INFORMATION, depending on whether ReadDirectoryChangesExW is
// available or not.
template <typename BufferType>
void ProcessDirChanges(void* buffer, size_t read_bytes)
ABSL_LOCKS_EXCLUDED(modified_files_mutex_) {
{
absl::MutexLock mutex(&modified_files_mutex_);
assert(buffer);
size_t offset = 0;
FileMap modified_files;
while (offset < read_bytes) {
auto* info = reinterpret_cast<BufferType*>(
reinterpret_cast<char*>(buffer) + offset);
std::string file_name = Util::WideToUtf8Str(std::wstring(
info->FileName,
info->FileName + (info->FileNameLength / sizeof(info->FileName))));
bool was_added = false;
auto iter = modified_files_.find(file_name);
if (iter != modified_files_.end())
was_added = iter->second.action == FileAction::kAdded;
// Merge the current change into |modified_files_| in a way so that
// sequences that use temp files (e.g. ADDED - MODIFIED - REMOVED)
// result in no entry in |modified_files_| at all.
bool remove = false;
FileAction new_action = FileAction::kAdded;
switch (info->Action) {
case FILE_ACTION_REMOVED:
case FILE_ACTION_RENAMED_OLD_NAME:
// If the entry was originally added, remove it again.
if (was_added)
remove = true;
else
new_action = FileAction::kDeleted;
break;
case FILE_ACTION_ADDED:
case FILE_ACTION_RENAMED_NEW_NAME:
new_action = FileAction::kAdded;
break;
case FILE_ACTION_MODIFIED:
// Keep the "added" state if the file was originally added.
new_action = was_added ? FileAction::kAdded : FileAction::kModified;
break;
}
// The file stats are only present if ReadDirectoryChangesEx is present.
// Otherwise, we'll get them later.
bool is_dir = false;
uint64_t size = 0;
int64_t mtime = 0;
if constexpr (std::is_same_v<BufferType,
FILE_NOTIFY_EXTENDED_INFORMATION>) {
is_dir = (info->FileAttributes & FILE_ATTRIBUTE_DIRECTORY) != 0;
size = static_cast<uint64_t>(info->FileSize.QuadPart);
mtime = ToUnixTime(info->LastModificationTime);
}
if (remove) {
modified_files_.erase(iter);
} else {
// Note: insert_or_assign updates earlier entries.
modified_files_.insert_or_assign(
file_name, FileInfo(new_action, is_dir, size, mtime));
}
++event_count_;
// No further entry exists.
if (info->NextEntryOffset == 0) break;
offset += info->NextEntryOffset;
}
}
// Invoke files changed callback if present.
if (files_changed_cb_) files_changed_cb_();
}
void SetStatus(const absl::Status& status)
ABSL_LOCKS_EXCLUDED(status_mutex_) {
LOG_DEBUG("Setting status '%s' of the file watcher process",
status.ToString().c_str());
absl::MutexLock mutex(&status_mutex_);
status_ = status;
}
// Modifies the file watcher state iff it is not shutting down.
void MaybeSetState(FileWatcherState state) ABSL_LOCKS_EXCLUDED(state_mutex_) {
LOG_DEBUG("Setting state %u of the file watcher process", state);
absl::MutexLock mutex(&state_mutex_);
if (state_ != FileWatcherState::kShuttingDown) state_ = state;
}
std::string dir_path_; // the path to the watched directory in UTF8.
ScopedHandle shutdown_event_; // event to shutdown the watcher.
std::thread dir_reader_; // watching thread.
mutable absl::Mutex status_mutex_;
absl::Status status_ = absl::OkStatus() ABSL_GUARDED_BY(status_mutex_);
mutable absl::Mutex modified_files_mutex_;
FileMap modified_files_ ABSL_GUARDED_BY(modified_files_mutex_);
uint32_t event_count_ ABSL_GUARDED_BY(modified_files_mutex_) = 0;
uint32_t dir_recreate_count_ ABSL_GUARDED_BY(modified_files_mutex_) = 0;
mutable absl::Mutex state_mutex_;
FileWatcherState state_ = FileWatcherState::kDefault ABSL_GUARDED_BY(
state_mutex_); // the current watcher state.
// Pointer to ReadDirectoryChangesExW function if available.
decltype(ReadDirectoryChangesExW)* read_directory_changes_ex_ = nullptr;
FilesChangedCb files_changed_cb_; // callback to react on modifications
// inside the watched directory.
DirRecreatedCb dir_recreated_cb_; // callback to react on the
// creation/removal/re-creation of the
// watched directory. It is not guarantee
// that the watched directory exists.
unsigned int timeout_ms_; // timeout in ms to wait for a change.
};
FileWatcherWin::FileWatcherWin(std::string directory) : dir_path_(directory) {}
FileWatcherWin::~FileWatcherWin() {
absl::Status status = StopWatching();
if (!status.ok()) {
LOG_WARNING("Failed to stop watching files: %s", status.ToString());
}
}
FileWatcherWin::FileMap FileWatcherWin::GetModifiedFiles() {
absl::MutexLock mutex(&modified_files_mutex_);
FileMap files;
modified_files_.swap(files);
if (async_watcher_) {
for (const auto& [path, info] : async_watcher_->GetModifiedFiles())
files.insert_or_assign(path, info);
}
return files;
}
absl::Status FileWatcherWin::StartWatching(FilesChangedCb files_changed_cb,
DirRecreatedCb dir_recreated_cb,
unsigned int timeout_ms) {
LOG_INFO("Starting the file watcher");
async_watcher_ = std::make_unique<AsyncFileWatcher>(
dir_path_, std::move(files_changed_cb), std::move(dir_recreated_cb),
timeout_ms, enforceLegacyReadDirectoryChangesForTesting_);
while (GetStatus().ok() && !IsWatching()) {
std::this_thread::sleep_for(std::chrono::milliseconds(1));
}
return GetStatus();
}
absl::Status FileWatcherWin::StopWatching() {
LOG_INFO("Stopping the file watcher");
if (!async_watcher_) {
return absl::OkStatus();
}
async_watcher_->Shutdown();
absl::MutexLock mutex(&modified_files_mutex_);
FileMap files = async_watcher_->GetModifiedFiles();
if (modified_files_.empty()) {
modified_files_.swap(files);
} else {
for (const auto& [path, info] : files)
modified_files_.insert_or_assign(path, info);
}
absl::Status async_status = async_watcher_->GetStatus();
async_watcher_.reset();
return async_status;
}
bool FileWatcherWin::IsWatching() const {
return async_watcher_ ? async_watcher_->IsWatching() : false;
}
absl::Status FileWatcherWin::GetStatus() const {
return async_watcher_ ? async_watcher_->GetStatus() : absl::OkStatus();
}
uint32_t FileWatcherWin::GetEventCountForTesting() const {
return async_watcher_ ? async_watcher_->GetEventCount() : 0;
}
uint32_t FileWatcherWin::GetDirRecreateEventCountForTesting() const {
return async_watcher_ ? async_watcher_->GetDirRecreateEventCount() : 0;
}
void FileWatcherWin::EnforceLegacyReadDirectoryChangesForTesting() {
assert(!IsWatching());
enforceLegacyReadDirectoryChangesForTesting_ = true;
}
} // namespace cdc_ft

111
common/file_watcher_win.h Normal file
View File

@@ -0,0 +1,111 @@
/*
* Copyright 2022 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef COMMON_FILE_WATCHER_WIN_H_
#define COMMON_FILE_WATCHER_WIN_H_
#define WIN32_LEAN_AND_MEAN
#include <string>
#include <unordered_map>
#include "absl/base/thread_annotations.h"
#include "absl/status/status.h"
#include "absl/synchronization/mutex.h"
namespace cdc_ft {
class AsyncFileWatcher;
// FileWatcherWin observes changes done in a specific directory on Windows.
class FileWatcherWin {
public:
// Default timeout in milliseconds.
static constexpr unsigned int kFileWatcherTimeoutMs = 1000;
using FilesChangedCb = std::function<void()>;
using DirRecreatedCb = std::function<void()>;
enum class FileAction { kAdded, kModified, kDeleted };
struct FileInfo {
FileAction action;
bool is_dir;
uint64_t size;
int64_t mtime;
FileInfo(FileAction action, bool is_dir, uint64_t size, int64_t mtime)
: action(action), is_dir(is_dir), size(size), mtime(mtime) {}
};
using FileMap = std::unordered_map<std::string, FileInfo>;
explicit FileWatcherWin(std::string directory);
FileWatcherWin(const FileWatcherWin& other) = delete;
FileWatcherWin& operator=(const FileWatcherWin& other) = delete;
~FileWatcherWin();
// Returns a map that maps relative paths of modified files to their file
// attributes.
FileMap GetModifiedFiles() ABSL_LOCKS_EXCLUDED(modified_files_mutex_);
// Starts watching directory changes.
// |files_changed_cb| is called on a background thread whenever files changed.
// |dir_recreated_cb| is called on a background thread whenever the watched
// directory was removed/created/re-created. It does not guarantee that the
// watched directory exists. The caller should check on its own and handle
// a case if the directory still does not exist. The callback shows that all
// outstanding changes are not valid anymore.
// |timeout_ms| is a timeout in ms for a change in the watched directory.
absl::Status StartWatching(FilesChangedCb files_changed_cb = FilesChangedCb(),
DirRecreatedCb dir_recreated_cb = DirRecreatedCb(),
unsigned int timeout_ms = kFileWatcherTimeoutMs);
// Stops watching directory changes.
absl::Status StopWatching() ABSL_LOCKS_EXCLUDED(modified_files_mutex_);
// Indicates whether a directory is currently watched.
bool IsWatching() const;
// Returns the watching status.
absl::Status GetStatus() const;
// Returns the total file changed events received so far.
// Returns 0 if the watcher is currently in stopped state.
uint32_t GetEventCountForTesting() const;
// Returns the total number of events for the directory changes received so
// far. Returns 0 if the watcher is currently in stopped state.
uint32_t GetDirRecreateEventCountForTesting() const;
// Enforces the use of ReadDirectoryChanges instead of ReadDirectoryChangesEx.
// Must be called before StartWatching.
void EnforceLegacyReadDirectoryChangesForTesting();
private:
std::string dir_path_;
std::unique_ptr<AsyncFileWatcher> async_watcher_;
absl::Mutex modified_files_mutex_;
FileMap modified_files_ ABSL_GUARDED_BY(modified_files_mutex_);
bool enforceLegacyReadDirectoryChangesForTesting_ = false;
};
} // namespace cdc_ft
#endif // COMMON_FILE_WATCHER_WIN_H_

609
common/file_watcher_win_test.cc Normal file
View File

@@ -0,0 +1,609 @@
// Copyright 2022 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "common/file_watcher_win.h"
#define WIN32_LEAN_AND_MEAN
#include <chrono>
#include <string>
#include <thread>
#include "absl/strings/match.h"
#include "common/buffer.h"
#include "common/log.h"
#include "common/path.h"
#include "common/platform.h"
#include "common/status_test_macros.h"
#include "common/util.h"
#include "gtest/gtest.h"
namespace cdc_ft {
const char* ActionToString(const FileWatcherWin::FileAction& fa) {
switch (fa) {
case FileWatcherWin::FileAction::kAdded:
return "ADDED";
case FileWatcherWin::FileAction::kModified:
return "MODIFIED";
case FileWatcherWin::FileAction::kDeleted:
return "DELETED";
default:
return "UNKNOWN";
}
}
std::ostream& operator<<(std::ostream& os,
const FileWatcherWin::FileMap& files) {
for (const auto& [path, fi] : files) {
os << "path=" << path << ", action=" << ActionToString(fi.action)
<< ", is_dir=" << fi.is_dir << ", mtime=" << fi.mtime
<< ", size=" << fi.size << std::endl;
}
return os;
}
namespace {
constexpr char kWatcherTestDir[] = "watcher_test_dir";
constexpr char kWatcherWatchedDir[] = "watcher_watched_dir";
constexpr char kFirstData[] = {10, 20, 30, 40, 50, 60, 70, 80, 90};
constexpr char kSecondData[] = {100, 101, 102, 103, 104, 105, 106, 107, 108,
100, 101, 102, 103, 104, 105, 106, 107, 108};
constexpr size_t kFirstDataSize = sizeof(kFirstData);
constexpr size_t kSecondDataSize = sizeof(kSecondData);
constexpr char kFirstFile[] = "first_test_file.txt";
constexpr char kSecondFile[] = "second_test_file.txt";
constexpr char kFirstDir[] = "first_test_dir";
constexpr char kSecondDir[] = "second_test_dir";
constexpr bool kFile = false;
constexpr bool kDir = true;
constexpr absl::Duration kWaitTimeout = absl::Seconds(5);
constexpr unsigned int kFWTimeout = 10;
using FileMap = FileWatcherWin::FileMap;
using FileAction = FileWatcherWin::FileAction;
using FileInfo = FileWatcherWin::FileInfo;
class FileWatcherParameterizedTest : public ::testing::TestWithParam<bool> {
public:
FileWatcherParameterizedTest() : watcher_(watcher_dir_path_) {
Log::Initialize(std::make_unique<ConsoleLog>(LogLevel::kInfo));
}
~FileWatcherParameterizedTest() { Log::Shutdown(); }
void SetUp() override {
legacyReadDirectoryChanges_ = GetParam();
if (legacyReadDirectoryChanges_)
watcher_.EnforceLegacyReadDirectoryChangesForTesting();
EXPECT_OK(path::RemoveDirRec(watcher_dir_path_));
EXPECT_OK(path::CreateDirRec(watcher_dir_path_));
}
void TearDown() override { EXPECT_OK(path::RemoveDirRec(watcher_dir_path_)); }
// True if ReadDirectoryChangesW should be enforced (instead of *ExW).
bool legacyReadDirectoryChanges_ = false;
protected:
void OnFilesChanged() {
absl::MutexLock lock(&files_changed_mutex_);
files_changed_ = true;
}
void OnDirRecreated() {
absl::MutexLock lock(&files_changed_mutex_);
dir_recreated_ = true;
}
bool WaitForChange(uint32_t min_event_count = 0) {
absl::MutexLock lock(&files_changed_mutex_);
bool changed = false;
do {
auto cond = [this]() { return files_changed_; };
files_changed_mutex_.AwaitWithTimeout(absl::Condition(&cond),
kWaitTimeout);
changed = files_changed_;
files_changed_ = false;
} while (changed && watcher_.GetEventCountForTesting() < min_event_count);
return changed;
}
bool WaitForDirRecreated(uint32_t min_event_count = 0) {
absl::MutexLock lock(&files_changed_mutex_);
bool changed = false;
do {
auto cond = [this]() { return dir_recreated_; };
files_changed_mutex_.AwaitWithTimeout(absl::Condition(&cond),
kWaitTimeout);
changed = dir_recreated_;
dir_recreated_ = false;
} while (changed &&
watcher_.GetDirRecreateEventCountForTesting() < min_event_count);
return changed;
}
FileMap GetChangedFiles(size_t number_of_files) {
FileMap modified_files;
// Wait for events, until they are processed.
while (modified_files.size() < number_of_files) {
EXPECT_TRUE(WaitForChange());
for (const auto& [path, info] : watcher_.GetModifiedFiles())
modified_files.insert_or_assign(path, info);
}
return modified_files;
}
void ExpectFile(const FileMap& modified_files, const std::string& path) {
EXPECT_TRUE(modified_files.find(path) != modified_files.end())
<< path << " is missing from " << std::endl
<< modified_files;
}
void ExpectFileInfo(const FileMap& modified_files, const std::string& path,
FileAction action, bool is_dir, uint64_t size) {
auto iter = modified_files.find(path);
EXPECT_TRUE(iter != modified_files.end())
<< path << " is missing from " << std::endl
<< modified_files;
if (iter != modified_files.end()) {
EXPECT_EQ(iter->second.action, action);
// is_dir and size are not available for the legacy ReadDirectoryChangesW,
// but that data isn't needed, anyway.
if (action != FileAction::kDeleted || !legacyReadDirectoryChanges_) {
EXPECT_EQ(iter->second.is_dir, is_dir);
EXPECT_EQ(iter->second.size, size);
}
// Don't bother checking mtime here, it's checked elsewhere.
}
}
const std::string test_dir_path_ =
path::Join(path::GetTempDir(), kWatcherTestDir);
const std::string watcher_dir_path_ =
path::Join(test_dir_path_, kWatcherWatchedDir);
const std::string first_file_path_ =
path::Join(watcher_dir_path_, kFirstFile);
const std::string second_file_path_ =
path::Join(watcher_dir_path_, kSecondFile);
const std::string first_dir_path_ = path::Join(watcher_dir_path_, kFirstDir);
const std::string second_dir_path_ =
path::Join(watcher_dir_path_, kSecondDir);
FileWatcherWin watcher_;
bool files_changed_ ABSL_GUARDED_BY(files_changed_mutex_) = false;
bool dir_recreated_ ABSL_GUARDED_BY(files_changed_mutex_) = false;
absl::Mutex files_changed_mutex_;
};
TEST_P(FileWatcherParameterizedTest, DirDoesNotExist) {
FileWatcherWin watcher("non-existing folder");
if (legacyReadDirectoryChanges_)
watcher_.EnforceLegacyReadDirectoryChangesForTesting();
EXPECT_NOT_OK(watcher.StartWatching([this]() { OnFilesChanged(); }));
EXPECT_FALSE(watcher.IsWatching());
absl::Status status = watcher.GetStatus();
EXPECT_NOT_OK(status);
EXPECT_TRUE(absl::IsFailedPrecondition(status));
EXPECT_TRUE(absl::StrContains(status.message(), "Could not start watching"));
}
TEST_P(FileWatcherParameterizedTest, CreateFile) {
EXPECT_OK(watcher_.StartWatching([this]() { OnFilesChanged(); }));
EXPECT_OK(path::WriteFile(first_file_path_, kFirstData, kFirstDataSize));
FileMap modified_files = GetChangedFiles(1u);
EXPECT_EQ(modified_files.size(), 1u);
ExpectFile(modified_files, kFirstFile);
EXPECT_OK(watcher_.StopWatching());
}
TEST_P(FileWatcherParameterizedTest, CreateFileDelayed) {
EXPECT_OK(watcher_.StartWatching([this]() { OnFilesChanged(); }));
std::this_thread::sleep_for(std::chrono::milliseconds(20));
EXPECT_OK(path::WriteFile(first_file_path_, kFirstData, kFirstDataSize));
FileMap modified_files = GetChangedFiles(1u);
EXPECT_EQ(modified_files.size(), 1u);
ExpectFile(modified_files, kFirstFile);
EXPECT_OK(watcher_.StopWatching());
}
TEST_P(FileWatcherParameterizedTest, CreateTwoFiles) {
EXPECT_OK(watcher_.StartWatching([this]() { OnFilesChanged(); }));
EXPECT_OK(path::WriteFile(first_file_path_, kFirstData, kFirstDataSize));
EXPECT_OK(path::WriteFile(second_file_path_, kFirstData, kFirstDataSize));
FileMap modified_files = GetChangedFiles(2u);
EXPECT_EQ(modified_files.size(), 2u);
ExpectFile(modified_files, kFirstFile);
ExpectFile(modified_files, kSecondFile);
EXPECT_OK(watcher_.StopWatching());
}
TEST_P(FileWatcherParameterizedTest, CreateDir) {
EXPECT_OK(watcher_.StartWatching([this]() { OnFilesChanged(); }));
EXPECT_OK(path::CreateDir(first_dir_path_));
FileMap modified_files = GetChangedFiles(1u);
EXPECT_EQ(modified_files.size(), 1u);
ExpectFile(modified_files, kFirstDir);
EXPECT_OK(watcher_.StopWatching());
}
TEST_P(FileWatcherParameterizedTest, RenameDir) {
EXPECT_OK(path::CreateDir(first_dir_path_));
EXPECT_OK(watcher_.StartWatching([this]() { OnFilesChanged(); }));
EXPECT_OK(path::RenameFile(first_dir_path_, second_dir_path_));
FileMap modified_files = GetChangedFiles(2u);
EXPECT_EQ(modified_files.size(), 2u);
ExpectFileInfo(modified_files, kFirstDir, FileAction::kDeleted, kDir, 0);
ExpectFileInfo(modified_files, kSecondDir, FileAction::kAdded, kDir, 0);
EXPECT_OK(watcher_.StopWatching());
}
TEST_P(FileWatcherParameterizedTest, RenameFile) {
EXPECT_OK(path::WriteFile(first_file_path_, kFirstData, kFirstDataSize));
EXPECT_OK(watcher_.StartWatching([this]() { OnFilesChanged(); }));
EXPECT_OK(path::RenameFile(first_file_path_, second_file_path_));
FileMap modified_files = GetChangedFiles(2u);
EXPECT_EQ(modified_files.size(), 2u);
ExpectFileInfo(modified_files, kFirstFile, FileAction::kDeleted, kFile,
kFirstDataSize);
ExpectFileInfo(modified_files, kSecondFile, FileAction::kAdded, kFile,
kFirstDataSize);
EXPECT_OK(watcher_.StopWatching());
}
TEST_P(FileWatcherParameterizedTest, RemoveDir) {
EXPECT_OK(path::CreateDir(first_dir_path_));
EXPECT_OK(watcher_.StartWatching([this]() { OnFilesChanged(); }));
EXPECT_OK(path::RemoveDirRec(first_dir_path_));
FileMap modified_files = GetChangedFiles(1u);
EXPECT_EQ(modified_files.size(), 1u);
ExpectFile(modified_files, kFirstDir);
EXPECT_OK(watcher_.StopWatching());
}
TEST_P(FileWatcherParameterizedTest, RemoveFile) {
EXPECT_OK(path::WriteFile(first_file_path_, kFirstData, kFirstDataSize));
EXPECT_OK(watcher_.StartWatching([this]() { OnFilesChanged(); }));
EXPECT_OK(path::RemoveFile(first_file_path_));
FileMap modified_files = GetChangedFiles(1u);
EXPECT_EQ(modified_files.size(), 1u);
ExpectFile(modified_files, kFirstFile);
EXPECT_OK(watcher_.StopWatching());
}
TEST_P(FileWatcherParameterizedTest, ChangeFile) {
EXPECT_OK(path::WriteFile(first_file_path_, kFirstData, kFirstDataSize));
EXPECT_OK(watcher_.StartWatching([this]() { OnFilesChanged(); }));
EXPECT_OK(path::WriteFile(first_file_path_, kSecondData, kSecondDataSize));
FileMap modified_files = GetChangedFiles(1u);
EXPECT_EQ(modified_files.size(), 1u);
ExpectFile(modified_files, kFirstFile);
EXPECT_OK(watcher_.StopWatching());
}
TEST_P(FileWatcherParameterizedTest, DirHierarchy) {
EXPECT_OK(watcher_.StartWatching([this]() { OnFilesChanged(); }));
std::vector<std::string> files = {"1.txt", "2.txt", "3.txt", "4.txt",
"5.txt", "6.txt", "7.txt", "8.txt"};
EXPECT_OK(path::CreateDir(first_dir_path_));
EXPECT_OK(path::CreateDir(second_dir_path_));
for (const std::string& file : files) {
for (const auto& path :
{first_dir_path_, second_dir_path_, watcher_dir_path_})
EXPECT_OK(
path::WriteFile(path::Join(path, file), kFirstData, kFirstDataSize));
}
FileMap modified_files = GetChangedFiles(files.size() * 3 + 2);
ASSERT_EQ(modified_files.size(), files.size() * 3 + 2);
for (const std::string& file : files) {
ExpectFile(modified_files, path::Join(kFirstDir, file));
ExpectFile(modified_files, path::Join(kSecondDir, file));
ExpectFile(modified_files, file);
}
ExpectFile(modified_files, kFirstDir);
ExpectFile(modified_files, kSecondDir);
EXPECT_OK(watcher_.StopWatching());
}
TEST_P(FileWatcherParameterizedTest, NoReadDirChanges) {
EXPECT_OK(watcher_.StartWatching([this]() { OnFilesChanged(); }));
EXPECT_OK(watcher_.StopWatching());
FileMap modified_files = GetChangedFiles(0u);
EXPECT_EQ(modified_files.size(), 0u);
}
TEST_P(FileWatcherParameterizedTest, RestartWatchingNoChanges) {
EXPECT_OK(watcher_.StartWatching([this]() { OnFilesChanged(); }));
EXPECT_OK(watcher_.StopWatching());
// Restart with reading.
EXPECT_OK(watcher_.StartWatching([this]() { OnFilesChanged(); }));
EXPECT_OK(watcher_.StopWatching());
// Restart without reading.
EXPECT_OK(watcher_.StartWatching([this]() { OnFilesChanged(); }));
EXPECT_OK(watcher_.StopWatching());
}
TEST_P(FileWatcherParameterizedTest, RestartWatchingWithChanges) {
EXPECT_OK(watcher_.StartWatching([this]() { OnFilesChanged(); }));
EXPECT_OK(path::WriteFile(first_file_path_, kFirstData, kFirstDataSize));
EXPECT_OK(watcher_.StopWatching());
// first_test_dir should not be in the modification set.
EXPECT_OK(path::CreateDir(first_dir_path_));
EXPECT_OK(watcher_.StartWatching([this]() { OnFilesChanged(); }));
EXPECT_OK(path::WriteFile(second_file_path_, kSecondData, kSecondDataSize));
FileMap modified_files = GetChangedFiles(2u);
EXPECT_EQ(modified_files.size(), 2u);
ExpectFile(modified_files, kFirstFile);
ExpectFile(modified_files, kSecondFile);
EXPECT_OK(watcher_.StopWatching());
}
TEST_P(FileWatcherParameterizedTest, ReadFileNoNotification) {
EXPECT_OK(path::WriteFile(first_file_path_, kFirstData, kFirstDataSize));
EXPECT_OK(watcher_.StartWatching([this]() { OnFilesChanged(); }));
Buffer data;
EXPECT_OK(path::ReadFile(first_file_path_, &data));
FileMap modified_files = GetChangedFiles(0u);
EXPECT_EQ(modified_files.size(), 0u);
EXPECT_OK(watcher_.StopWatching());
}
TEST_P(FileWatcherParameterizedTest, SearchFilesNoNotification) {
EXPECT_OK(path::WriteFile(first_file_path_, kFirstData, kFirstDataSize));
EXPECT_OK(watcher_.StartWatching([this]() { OnFilesChanged(); }));
unsigned int counter = 0;
auto handler = [&counter](const std::string& /*dir*/,
const std::string& /*filename*/,
int64_t /*modified_time*/, uint64_t /*size*/,
bool /*is_directory*/) {
++counter;
return absl::OkStatus();
};
EXPECT_OK(path::SearchFiles(first_file_path_, true, handler));
EXPECT_EQ(counter, 1u);
FileMap modified_files = GetChangedFiles(0u);
EXPECT_EQ(modified_files.size(), 0u);
EXPECT_OK(watcher_.StopWatching());
}
TEST_P(FileWatcherParameterizedTest, ActionAdd) {
EXPECT_OK(watcher_.StartWatching([this]() { OnFilesChanged(); }));
EXPECT_OK(path::WriteFile(first_file_path_, kFirstData, kFirstDataSize));
EXPECT_TRUE(WaitForChange(/*min_event_count=*/2)); // 1x add, 1x modify
FileMap modified_files = watcher_.GetModifiedFiles();
EXPECT_EQ(modified_files.size(), 1u);
ExpectFileInfo(modified_files, kFirstFile, FileAction::kAdded, kFile,
kFirstDataSize);
EXPECT_OK(watcher_.StopWatching());
}
TEST_P(FileWatcherParameterizedTest, ActionModify) {
EXPECT_OK(path::WriteFile(first_file_path_, kFirstData, kFirstDataSize));
EXPECT_OK(watcher_.StartWatching([this]() { OnFilesChanged(); }));
EXPECT_OK(path::WriteFile(first_file_path_, kSecondData, kSecondDataSize));
EXPECT_TRUE(WaitForChange(/*min_event_count=*/2)); // 2x modify
FileMap modified_files = watcher_.GetModifiedFiles();
EXPECT_EQ(modified_files.size(), 1u);
ExpectFileInfo(modified_files, kFirstFile, FileAction::kModified, kFile,
kSecondDataSize);
EXPECT_OK(watcher_.StopWatching());
}
TEST_P(FileWatcherParameterizedTest, ActionAddModify) {
EXPECT_OK(watcher_.StartWatching([this]() { OnFilesChanged(); }));
EXPECT_OK(path::WriteFile(first_file_path_, kFirstData, kFirstDataSize));
EXPECT_OK(path::WriteFile(first_file_path_, kSecondData, kSecondDataSize));
EXPECT_TRUE(WaitForChange(/*min_event_count=*/4)); // 1x add, 3x modify
// Add + modify should not result in FileAction::kModified.
FileMap modified_files = watcher_.GetModifiedFiles();
EXPECT_EQ(modified_files.size(), 1u);
ExpectFileInfo(modified_files, kFirstFile, FileAction::kAdded, kFile,
kSecondDataSize);
EXPECT_OK(watcher_.StopWatching());
}
TEST_P(FileWatcherParameterizedTest, ActionDelete) {
EXPECT_OK(path::WriteFile(first_file_path_, kFirstData, kFirstDataSize));
EXPECT_OK(watcher_.StartWatching([this]() { OnFilesChanged(); }));
EXPECT_OK(path::RemoveFile(first_file_path_));
EXPECT_TRUE(WaitForChange(/*min_event_count=*/1)); // 1x remove
FileMap modified_files = watcher_.GetModifiedFiles();
EXPECT_EQ(modified_files.size(), 1u);
ExpectFileInfo(modified_files, kFirstFile, FileAction::kDeleted, kFile,
kFirstDataSize);
EXPECT_OK(watcher_.StopWatching());
}
TEST_P(FileWatcherParameterizedTest, ActionAddModifyRemove) {
EXPECT_OK(watcher_.StartWatching([this]() { OnFilesChanged(); }));
EXPECT_OK(path::WriteFile(first_file_path_, kFirstData, kFirstDataSize));
EXPECT_OK(path::WriteFile(first_file_path_, kSecondData, kSecondDataSize));
EXPECT_OK(path::RemoveFile(first_file_path_));
EXPECT_TRUE(
WaitForChange(/*min_event_count=*/5)); // 1x add, 3x modify, 1x remove
// The watcher should collapse add-modify-remove sequences and report no
// changes.
FileMap modified_files = watcher_.GetModifiedFiles();
EXPECT_EQ(modified_files.size(), 0u);
EXPECT_OK(watcher_.StopWatching());
}
TEST_P(FileWatcherParameterizedTest, ActionModifyRemove) {
EXPECT_OK(path::WriteFile(first_file_path_, kFirstData, kFirstDataSize));
EXPECT_OK(watcher_.StartWatching([this]() { OnFilesChanged(); }));
EXPECT_OK(path::WriteFile(first_file_path_, kSecondData, kSecondDataSize));
EXPECT_OK(path::RemoveFile(first_file_path_));
EXPECT_TRUE(WaitForChange(/*min_event_count=*/3)); // 2x modify, 1x remove
FileMap modified_files = watcher_.GetModifiedFiles();
EXPECT_EQ(modified_files.size(), 1u);
ExpectFileInfo(modified_files, kFirstFile, FileAction::kDeleted, kFile,
kSecondDataSize);
EXPECT_OK(watcher_.StopWatching());
}
TEST_P(FileWatcherParameterizedTest, ModifiedTime) {
EXPECT_OK(watcher_.StartWatching([this]() { OnFilesChanged(); }));
EXPECT_OK(path::WriteFile(first_file_path_, kFirstData, kFirstDataSize));
EXPECT_TRUE(WaitForChange(/*min_event_count=*/2)); // 2x modify
FileMap modified_files = watcher_.GetModifiedFiles();
ASSERT_EQ(modified_files.size(), 1u);
time_t mtime;
EXPECT_OK(path::GetFileTime(first_file_path_, &mtime));
const FileInfo& info = modified_files.begin()->second;
EXPECT_EQ(info.mtime, mtime);
EXPECT_OK(watcher_.StopWatching());
}
TEST_P(FileWatcherParameterizedTest, DeleteWatchedDir) {
EXPECT_OK(watcher_.StartWatching([this]() { OnFilesChanged(); },
[this]() { OnDirRecreated(); }));
EXPECT_OK(path::RemoveDirRec(watcher_dir_path_));
EXPECT_TRUE(WaitForDirRecreated(1u));
EXPECT_TRUE(watcher_.GetModifiedFiles().empty());
EXPECT_NOT_OK(watcher_.GetStatus());
// The error status should not be overwritten.
EXPECT_NOT_OK(watcher_.StopWatching());
}
TEST_P(FileWatcherParameterizedTest, RecreateWatchedDir) {
EXPECT_OK(watcher_.StartWatching([this]() { OnFilesChanged(); },
[this]() { OnDirRecreated(); }, kFWTimeout));
EXPECT_OK(path::RemoveDirRec(watcher_dir_path_));
EXPECT_TRUE(WaitForDirRecreated(1u));
EXPECT_OK(path::CreateDirRec(watcher_dir_path_));
EXPECT_TRUE(WaitForDirRecreated(2u));
EXPECT_TRUE(watcher_.GetModifiedFiles().empty());
EXPECT_OK(watcher_.GetStatus());
// Creation of a new file should be detected.
EXPECT_OK(path::WriteFile(first_file_path_, kFirstData, kFirstDataSize));
FileMap modified_files = GetChangedFiles(1u);
EXPECT_EQ(modified_files.size(), 1u);
ExpectFile(modified_files, kFirstFile);
EXPECT_OK(watcher_.StopWatching());
}
TEST_P(FileWatcherParameterizedTest, RecreateUpperDir) {
EXPECT_OK(watcher_.StartWatching([this]() { OnFilesChanged(); },
[this]() { OnDirRecreated(); }, kFWTimeout));
// Initially, there should be no dir_recreated_events.
EXPECT_EQ(watcher_.GetDirRecreateEventCountForTesting(), 0u);
EXPECT_OK(path::RemoveDirRec(test_dir_path_));
EXPECT_TRUE(WaitForDirRecreated(1u));
// Only 1 event should exist (for directory removal).
EXPECT_OK(path::CreateDirRec(test_dir_path_));
EXPECT_OK(path::CreateDirRec(watcher_dir_path_));
// As the upper level directory is created separately, no new event should be
// available.
EXPECT_TRUE(WaitForDirRecreated(2u));
// Only 1 additional event should be registered for creation of the directory.
EXPECT_EQ(watcher_.GetDirRecreateEventCountForTesting(), 2u);
EXPECT_TRUE(watcher_.GetModifiedFiles().empty());
EXPECT_OK(watcher_.GetStatus());
// Creation of a new file should be detected.
EXPECT_OK(path::WriteFile(first_file_path_, kFirstData, kFirstDataSize));
FileMap modified_files = GetChangedFiles(1u);
EXPECT_EQ(modified_files.size(), 1u);
ExpectFile(modified_files, kFirstFile);
// No new events should be registered for the watched directory.
EXPECT_EQ(watcher_.GetDirRecreateEventCountForTesting(), 2u);
EXPECT_OK(watcher_.StopWatching());
}
TEST_P(FileWatcherParameterizedTest, RecreateWatchedDirNoOldChanges) {
EXPECT_OK(watcher_.StartWatching([this]() { OnFilesChanged(); },
[this]() { OnDirRecreated(); }, kFWTimeout));
EXPECT_OK(path::WriteFile(first_file_path_, kFirstData, kFirstDataSize));
EXPECT_OK(path::RemoveDirRec(watcher_dir_path_));
EXPECT_OK(path::CreateDirRec(watcher_dir_path_));
EXPECT_TRUE(WaitForDirRecreated(2u));
EXPECT_TRUE(watcher_.GetModifiedFiles().empty());
EXPECT_OK(watcher_.GetStatus());
EXPECT_OK(watcher_.StopWatching());
}
struct TestName {
std::string operator()(const testing::TestParamInfo<bool>& legacy) const {
return legacy.param ? "ReadDirectoryChangesW" : "ReadDirectoryChangesExW";
}
};
// Run without and with legacy ReadDirectoryChangesW function.
INSTANTIATE_TEST_CASE_P(FileWatcherTest, FileWatcherParameterizedTest,
::testing::Values(false, true), TestName());
} // namespace
} // namespace cdc_ft

92
common/gamelet_component.cc Normal file
View File

@@ -0,0 +1,92 @@
// Copyright 2022 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "common/gamelet_component.h"
#include <cinttypes>
#include "absl/strings/str_format.h"
#include "absl/strings/str_split.h"
#include "common/path.h"
#include "common/status.h"
namespace cdc_ft {
GameletComponent::GameletComponent(std::string filename, uint64_t size,
time_t modified_time)
: filename(filename), size(size), modified_time(modified_time) {}
GameletComponent::~GameletComponent() = default;
bool GameletComponent::operator==(const GameletComponent& other) const {
return filename == other.filename && size == other.size &&
modified_time == other.modified_time;
}
bool GameletComponent::operator!=(const GameletComponent& other) const {
return !(*this == other);
}
// static
absl::Status GameletComponent::Get(
const std::vector<std::string> component_paths,
std::vector<GameletComponent>* components) {
components->clear();
for (const std::string& path : component_paths) {
path::Stats stats;
absl::Status status = path::GetStats(path, &stats);
if (!status.ok())
return WrapStatus(status, "GetStats() failed for '%s'", path);
components->emplace_back(path::BaseName(path), stats.size,
stats.modified_time);
}
return absl::OkStatus();
}
// static
std::string GameletComponent::ToCommandLineArgs(
const std::vector<GameletComponent>& components) {
std::string args;
for (const GameletComponent& comp : components) {
args +=
absl::StrFormat("%s%s %u %d", args.empty() ? "" : " ",
comp.filename.c_str(), comp.size, comp.modified_time);
}
return args;
}
// static
std::vector<GameletComponent> GameletComponent::FromCommandLineArgs(
int argc, const char** argv) {
std::vector<GameletComponent> components;
for (int n = 0; n + 2 < argc; n += 3) {
components.emplace_back(argv[n], std::stol(argv[n + 1]),
std::stol(argv[n + 2]));
}
return components;
}
// static
std::vector<GameletComponent> GameletComponent::FromCommandLineArgs(
const std::string& components_arg) {
std::vector<std::string> args_vec = absl::StrSplit(components_arg, ' ');
int argc = static_cast<int>(args_vec.size());
std::vector<const char*> argv;
for (const std::string& arg : args_vec) argv.push_back(arg.c_str());
return FromCommandLineArgs(argc, argv.data());
}
} // namespace cdc_ft

70
common/gamelet_component.h Normal file
View File

@@ -0,0 +1,70 @@
/*
* Copyright 2022 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef COMMON_GAMELET_COMPONENT_H_
#define COMMON_GAMELET_COMPONENT_H_
#include <string>
#include <vector>
#include "absl/status/status.h"
namespace cdc_ft {
// Provides functionality to check the freshness of gamelet components.
// The components are considered fresh if both the timestamp and the file size
// match.
struct GameletComponent {
std::string filename;
uint64_t size;
int64_t modified_time;
GameletComponent(std::string filename, uint64_t size, time_t modified_time);
~GameletComponent();
bool operator==(const GameletComponent& other) const;
bool operator!=(const GameletComponent& other) const;
// Gets the list of gamelet |components| from the given |component_paths|.
// Returns false if any of the components does not exist or cannot be stat'ed.
static absl::Status Get(const std::vector<std::string> component_paths,
std::vector<GameletComponent>* components);
// Serializes the list of gamelet |components| as command line options.
// This should be called on the workstation. The resulting command line
// should be passed to the gamelet process.
static std::string ToCommandLineArgs(
const std::vector<GameletComponent>& components);
// Deserializes the list of gamelet components from command line options.
// This should be called in the gamelet process to retrieve the workstation
// components, which can then be compared to the gamelet components to check
// freshness.
static std::vector<GameletComponent> FromCommandLineArgs(int argc,
const char** argv);
// Deserializes the list of gamelet components from a |components_arg| string
// as it is returned by ToCommandLineArgs().
// This should be called in the gamelet process to retrieve the workstation
// components, which can then be compared to the gamelet components to check
// freshness.
static std::vector<GameletComponent> FromCommandLineArgs(
const std::string& components_arg);
};
} // namespace cdc_ft
#endif // COMMON_GAMELET_COMPONENT_H_

124
common/gamelet_component_test.cc Normal file
View File

@@ -0,0 +1,124 @@
// Copyright 2022 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "common/gamelet_component.h"
#include "absl/strings/str_split.h"
#include "common/log.h"
#include "common/path.h"
#include "common/status_test_macros.h"
#include "common/test_main.h"
#include "gtest/gtest.h"
namespace cdc_ft {
namespace {
class GameletComponentTest : public ::testing::Test {
public:
void SetUp() override {
Log::Initialize(std::make_unique<ConsoleLog>(LogLevel::kInfo));
}
void TearDown() override { Log::Shutdown(); }
protected:
std::string base_dir_ = GetTestDataDir("gamelet_component");
std::string valid_component_path_ =
path::Join(base_dir_, "valid", "cdc_rsync_server");
std::string invalid_component_path_ =
path::Join(base_dir_, "invalid", "cdc_rsync_server");
std::string other_component_path_ =
path::Join(base_dir_, "other", "cdc_rsync_server");
};
TEST_F(GameletComponentTest, EqualityOperators) {
constexpr uint64_t size1 = 1001;
constexpr uint64_t size2 = 1002;
constexpr int64_t modified_time1 = 5001;
constexpr int64_t modified_time2 = 5002;
GameletComponent a("file1", size1, modified_time1);
GameletComponent b = a;
EXPECT_TRUE(a == b && !(a != b));
b.filename = "file2";
EXPECT_TRUE(!(a == b) && a != b);
b = a;
b.size = size2;
EXPECT_TRUE(!(a == b) && a != b);
b = a;
b.modified_time = modified_time2;
EXPECT_TRUE(!(a == b) && a != b);
}
TEST_F(GameletComponentTest, GetValidComponents) {
std::vector<GameletComponent> components;
EXPECT_OK(GameletComponent::Get({valid_component_path_}, &components));
ASSERT_EQ(components.size(), 1);
EXPECT_EQ(components[0].filename, "cdc_rsync_server");
EXPECT_GT(components[0].size, 0);
EXPECT_GT(components[0].modified_time, 0);
}
TEST_F(GameletComponentTest, GetInvalidComponents) {
std::vector<GameletComponent> components;
EXPECT_NOT_OK(GameletComponent::Get({invalid_component_path_}, &components));
}
TEST_F(GameletComponentTest, GetChangedComponents) {
std::vector<GameletComponent> components;
EXPECT_OK(GameletComponent::Get({valid_component_path_}, &components));
std::vector<GameletComponent> other_components;
EXPECT_OK(GameletComponent::Get({other_component_path_}, &other_components));
// Force equal timestamps, so that we don't depend on when the files were
// actually written to everyone's drives.
ASSERT_EQ(components.size(), other_components.size());
for (size_t n = 0; n < components.size(); ++n) {
other_components[n].modified_time = components[n].modified_time;
EXPECT_NE(components, other_components);
}
}
TEST_F(GameletComponentTest, Serialization) {
std::vector<GameletComponent> components;
EXPECT_OK(GameletComponent::Get({valid_component_path_}, &components));
std::string args = GameletComponent::ToCommandLineArgs(components);
// FromCommandLineArgs() for a single string arg.
std::vector<GameletComponent> deserialized_components =
GameletComponent::FromCommandLineArgs(args);
EXPECT_EQ(components, deserialized_components);
// FromCommandLineArgs() for argc/argv.
std::vector<std::string> args_vec = absl::StrSplit(args, ' ');
int argc = static_cast<int>(args_vec.size());
std::vector<const char*> argv;
for (const std::string& arg : args_vec) argv.push_back(arg.c_str());
deserialized_components =
GameletComponent::FromCommandLineArgs(argc, argv.data());
EXPECT_EQ(components, deserialized_components);
}
} // namespace
} // namespace cdc_ft

55
common/grpc_status.h Normal file
View File

@@ -0,0 +1,55 @@
/*
* Copyright 2022 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef COMMON_GRPC_STATUS_H_
#define COMMON_GRPC_STATUS_H_
#include "absl/status/status.h"
#include "grpcpp/grpcpp.h"
namespace cdc_ft {
//
// gRPC status conversion convenience methods.
//
inline grpc::Status ToGrpcStatus(const absl::Status& status) {
if (status.ok()) return grpc::Status::OK;
return grpc::Status(static_cast<grpc::StatusCode>(status.code()),
std::string(status.message()));
}
inline absl::Status ToAbslStatus(const grpc::Status& status) {
if (status.ok()) return absl::OkStatus();
return absl::Status(static_cast<absl::StatusCode>(status.error_code()),
std::string(status.error_message()));
}
#define RETURN_GRPC_IF_ERROR(expr) \
do { \
absl::Status __absl_status = (expr); \
if (!__absl_status.ok()) return ToGrpcStatus(__absl_status); \
} while (0)
#define RETURN_ABSL_IF_ERROR(expr) \
do { \
grpc::Status __grpc_status = (expr); \
if (!__grpc_status.ok()) return ToAbslStatus(__grpc_status); \
} while (0)
} // namespace cdc_ft
#endif // COMMON_GRPC_STATUS_H_

170
common/log.cc Normal file
View File

@@ -0,0 +1,170 @@
// Copyright 2022 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "log.h"
#include <cassert>
#include "common/platform.h"
#if PLATFORM_WINDOWS
#define WIN32_LEAN_AND_MEAN
#include <windows.h>
#endif
namespace cdc_ft {
namespace {
const char* GetLogLevelString(LogLevel level) {
switch (level) {
case LogLevel::kVerbose:
return "VERBOSE";
case LogLevel::kDebug:
return "DEBUG";
case LogLevel::kInfo:
return "INFO";
case LogLevel::kWarning:
return "WARNING";
case LogLevel::kError:
return "ERROR";
}
return "UnknownLogLevel";
}
} // namespace
// static
void Log::Initialize(std::unique_ptr<Log> log) {
assert(!instance_);
instance_ = log.release();
}
// static
void Log::Shutdown() {
assert(instance_);
delete instance_;
instance_ = nullptr;
}
// static
Log* Log::Instance() {
assert(instance_);
return instance_;
}
// static
Log* Log::MaybeNullInstance() { return instance_; }
Log* Log::instance_ = nullptr;
Log::~Log() = default;
// static
LogLevel Log::VerbosityToLogLevel(int verbosity) {
if (verbosity >= 4) {
return LogLevel::kVerbose;
}
if (verbosity >= 3) {
return LogLevel::kDebug;
}
if (verbosity >= 2) {
return LogLevel::kInfo;
}
return LogLevel::kWarning;
}
// static
void Log::DefaultWriteLogMessage(LogLevel level, const char* file, int line,
const char* message) {
// Only print warnings and above.
if (level < LogLevel::kWarning) return;
fprintf(stderr, "%-7s %s(%i): %s\n", GetLogLevelString(level), file, line,
message);
}
#if PLATFORM_WINDOWS
enum Colors {
kLightGray = 7,
kGray = 8,
kBlue = 9,
kCyan = 11,
kRed = 12,
kYellow = 14,
kWhite = 15
};
WORD GetConsoleColor(LogLevel level) {
switch (level) {
case LogLevel::kVerbose:
return kGray;
case LogLevel::kDebug:
return kCyan;
case LogLevel::kInfo:
return kWhite;
case LogLevel::kWarning:
return kYellow;
case LogLevel::kError:
return kRed;
}
return 15;
}
#endif
void ConsoleLog::WriteLogMessage(LogLevel level, const char* file, int line,
const char* func, const char* message) {
absl::MutexLock lock(&mutex_);
// Show leaner log messages in non-verbose mode.
bool show_file_func = GetLogLevel() <= LogLevel::kDebug;
FILE* stdfile = level >= LogLevel::kError ? stderr : stdout;
#if PLATFORM_WINDOWS
HANDLE hConsole = GetStdHandle(STD_OUTPUT_HANDLE);
SetConsoleTextAttribute(hConsole, GetConsoleColor(level));
if (show_file_func) {
fprintf(stdfile, "%s(%i): %s(): %s\n", file, line, func, message);
} else {
fprintf(stdfile, "%s\n", message);
}
SetConsoleTextAttribute(hConsole, kLightGray);
#else
if (show_file_func) {
fprintf(stdfile, "%-7s %s(%i): %s(): %s\n", GetLogLevelString(level), file,
line, func, message);
} else {
fprintf(stdfile, "%-7s %s\n", GetLogLevelString(level), message);
}
#endif
}
FileLog::FileLog(LogLevel log_level, const char* path) : Log(log_level) {
file_ = fopen(path, "wt");
if (!file_) fprintf(stderr, "Failed to open log file '%s'", path);
}
FileLog::~FileLog() {
if (file_) fclose(file_);
}
void FileLog::WriteLogMessage(LogLevel level, const char* file, int line,
const char* func, const char* message) {
if (!file_) return;
std::string timestamp = clock_.FormatNow("%Y-%m-%d %H:%M:%S.", true);
absl::MutexLock lock(&mutex_);
fprintf(file_, "%s %-7s %s(%i): %s(): %s\n", timestamp.c_str(),
GetLogLevelString(level), file, line, func, message);
fflush(file_);
}
} // namespace cdc_ft

144
common/log.h Normal file
View File

@@ -0,0 +1,144 @@
/*
* Copyright 2022 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef COMMON_LOG_H_
#define COMMON_LOG_H_
#include <memory>
#include "absl/strings/str_format.h"
#include "absl/synchronization/mutex.h"
#include "common/clock.h"
namespace cdc_ft {
enum class LogLevel { kVerbose, kDebug, kInfo, kWarning, kError };
// Note: Bazel always uses forward slashes for paths.
#define __FILENAME__ \
(strrchr(__FILE__, '/') ? strrchr(__FILE__, '/') + 1 : __FILE__)
#define LOG_VERBOSE(...) \
::cdc_ft::Log::SafePrintf(::cdc_ft::LogLevel::kVerbose, __FILENAME__, \
__LINE__, __func__, __VA_ARGS__);
#define LOG_DEBUG(...) \
::cdc_ft::Log::SafePrintf(::cdc_ft::LogLevel::kDebug, __FILENAME__, \
__LINE__, __func__, __VA_ARGS__);
#define LOG_INFO(...) \
::cdc_ft::Log::SafePrintf(::cdc_ft::LogLevel::kInfo, __FILENAME__, __LINE__, \
__func__, __VA_ARGS__);
#define LOG_WARNING(...) \
::cdc_ft::Log::SafePrintf(::cdc_ft::LogLevel::kWarning, __FILENAME__, \
__LINE__, __func__, __VA_ARGS__);
#define LOG_ERROR(...) \
::cdc_ft::Log::SafePrintf(::cdc_ft::LogLevel::kError, __FILENAME__, \
__LINE__, __func__, __VA_ARGS__);
#define LOG_LEVEL(level, ...) \
::cdc_ft::Log::SafePrintf(level, __FILENAME__, __LINE__, __func__, \
__VA_ARGS__);
class Log {
public:
// Initializes the Log singleton. Should be called in the beginning.
static void Initialize(std::unique_ptr<Log> log);
// Deletes the Log singleton. Must be called in the end.
static void Shutdown();
// Returns the global log instance.
static Log* Instance();
template <typename... Args>
static void SafePrintf(LogLevel level, const char* file, int line,
const char* func,
const absl::FormatSpec<Args...>& format,
Args... args) {
Log* inst = MaybeNullInstance();
if (inst) {
inst->Printf(level, file, line, func, format, args...);
} else {
DefaultWriteLogMessage(level, file, line,
absl::StrFormat(format, args...).c_str());
}
}
template <typename... Args>
void Printf(LogLevel level, const char* file, int line, const char* func,
const absl::FormatSpec<Args...>& format, Args... args) {
if (log_level_ <= level) {
WriteLogMessage(level, file, line, func,
absl::StrFormat(format, args...).c_str());
}
}
static LogLevel VerbosityToLogLevel(int verbosity);
void SetLogLevel(LogLevel log_level) { log_level_ = log_level; }
LogLevel GetLogLevel() const { return log_level_; }
virtual ~Log();
protected:
explicit Log(LogLevel log_level) : log_level_(log_level) {}
virtual void WriteLogMessage(LogLevel level, const char* file, int line,
const char* func, const char* message) = 0;
private:
// A default log writer implementation in case the log was not initialized.
static void DefaultWriteLogMessage(LogLevel level, const char* file, int line,
const char* message);
// Returns the global log instance.
static Log* MaybeNullInstance();
static Log* instance_;
LogLevel log_level_;
};
class ConsoleLog : public Log {
public:
explicit ConsoleLog(LogLevel log_level) : Log(log_level) {}
protected:
void WriteLogMessage(LogLevel level, const char* file, int line,
const char* func, const char* message) override
ABSL_LOCKS_EXCLUDED(mutex_);
private:
absl::Mutex mutex_;
};
class FileLog : public Log {
public:
FileLog(LogLevel log_level, const char* path);
~FileLog();
protected:
void WriteLogMessage(LogLevel level, const char* file, int line,
const char* func, const char* message) override
ABSL_LOCKS_EXCLUDED(mutex_);
private:
FILE* file_;
DefaultSystemClock clock_;
absl::Mutex mutex_;
};
} // namespace cdc_ft
#endif // COMMON_LOG_H_

134
common/log_test.cc Normal file
View File

@@ -0,0 +1,134 @@
// Copyright 2022 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "common/log.h"
#include "absl/strings/match.h"
#include "common/path.h"
#include "common/status_test_macros.h"
#include "gtest/gtest.h"
namespace cdc_ft {
namespace {
constexpr char kLogMessage[] = "Test log";
class TestLog : public Log {
public:
explicit TestLog(LogLevel log_level) : Log(log_level) {}
LogLevel LastLevel() const { return last_level_; }
const std::string& LastFunc() const { return last_func_; }
const std::string& LastMessage() const { return last_message_; }
protected:
void WriteLogMessage(LogLevel level, const char* file, int line,
const char* func, const char* message) override {
constexpr char expected_file[] = "log_test.cc";
// The full filename depends on the compiler invocation, so we just compare
// the basename.
EXPECT_TRUE(absl::StrContains(file, expected_file))
<< "File name '" << file << "' does not contain expected name '"
<< expected_file << "'.";
EXPECT_GT(line, 0);
last_level_ = level;
last_func_ = func;
last_message_ = message;
}
private:
LogLevel last_level_;
std::string last_func_;
std::string last_message_;
};
class LogTest : public ::testing::Test {
public:
void SetUp() override {
Log::Initialize(std::make_unique<TestLog>(LogLevel::kInfo));
}
void TearDown() override { Log::Shutdown(); }
protected:
TestLog* Log() const { return static_cast<TestLog*>(Log::Instance()); }
};
TEST_F(LogTest, LogMessage) {
LOG_INFO(kLogMessage);
EXPECT_EQ(kLogMessage, Log()->LastMessage());
}
TEST_F(LogTest, LogFormattedMessage) {
LOG_INFO("Test %i %s", 123, "message");
EXPECT_EQ("Test 123 message", Log()->LastMessage());
}
TEST_F(LogTest, IgnoresBelowLogLevel) {
EXPECT_EQ(Log()->GetLogLevel(), LogLevel::kInfo);
LOG_DEBUG(kLogMessage);
EXPECT_EQ("", Log()->LastMessage());
Log()->SetLogLevel(LogLevel::kDebug);
LOG_DEBUG(kLogMessage);
EXPECT_EQ(kLogMessage, Log()->LastMessage());
}
TEST_F(LogTest, VerbosityToLogLevel) {
EXPECT_EQ(Log::VerbosityToLogLevel(0), LogLevel::kWarning);
EXPECT_EQ(Log::VerbosityToLogLevel(1), LogLevel::kWarning);
EXPECT_EQ(Log::VerbosityToLogLevel(2), LogLevel::kInfo);
EXPECT_EQ(Log::VerbosityToLogLevel(3), LogLevel::kDebug);
EXPECT_EQ(Log::VerbosityToLogLevel(4), LogLevel::kVerbose);
EXPECT_EQ(Log::VerbosityToLogLevel(5), LogLevel::kVerbose);
}
TEST(FileLogTest, LogToFile) {
std::string tmp_dir = path::GetTempDir();
std::string log_path = path::Join(tmp_dir, "__log_unittest.log");
Log::Initialize(std::make_unique<FileLog>(LogLevel::kInfo, log_path.c_str()));
LOG_ERROR("Error");
LOG_INFO("Info");
LOG_DEBUG("Debug");
Log::Shutdown();
std::vector<std::string> lines;
ASSERT_OK(
path::ReadAllLines(log_path, &lines, path::ReadFlags::kRemoveEmpty));
ASSERT_EQ(lines.size(), 2);
EXPECT_TRUE(absl::StrContains(lines[0], "ERROR")) << lines[0];
EXPECT_TRUE(absl::StrContains(lines[0], "log_test.cc")) << lines[0];
EXPECT_TRUE(absl::StrContains(lines[0], "Error")) << lines[0];
EXPECT_TRUE(absl::StrContains(lines[1], "INFO")) << lines[1];
EXPECT_TRUE(absl::StrContains(lines[1], "log_test.cc")) << lines[1];
EXPECT_TRUE(absl::StrContains(lines[1], "Info")) << lines[1];
// Check date and time.
int yy, mo, da, hh, mm, ss, msec;
int scan_res = sscanf(lines[0].c_str(), "%04d-%02d-%02d %02d:%02d:%02d.%03d",
&yy, &mo, &da, &hh, &mm, &ss, &msec);
EXPECT_EQ(scan_res, 7) << lines[0];
}
TEST(NoLogTest, LogNotInitialized) {
// Using the log before initializing it should not trigger an assertion.
LOG_ERROR("Error");
LOG_INFO("Info");
LOG_DEBUG("Debug");
}
} // namespace
} // namespace cdc_ft

1190
common/path.cc Normal file
View File

File diff suppressed because it is too large Load Diff

417
common/path.h Normal file
View File

@@ -0,0 +1,417 @@
/*
* Copyright 2022 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef COMMON_PATH_H_
#define COMMON_PATH_H_
#include <cstdio>
#include <functional>
#include <string>
#include <vector>
#include "absl/status/status.h"
#include "absl/status/statusor.h"
#include "absl/strings/string_view.h"
#include "common/platform.h"
#if PLATFORM_LINUX
#define feof_nolock feof_unlocked
#define fread_nolock fread_unlocked
#define fseek64 fseeko
#define fseek64_nolock fseeko
#define ftell64 ftello
#define fwrite_nolock fwrite_unlocked
#elif PLATFORM_WINDOWS
#define feof_nolock feof
#define fread_nolock _fread_nolock
#define fseek64 _fseeki64
#define fseek64_nolock _fseeki64_nolock
#define ftell64 _ftelli64
#define fwrite_nolock _fwrite_nolock
#endif
namespace cdc_ft {
class Buffer;
namespace path {
// Returns the native path separator character for this platform.
inline char PathSeparator() {
#ifdef PLATFORM_WINDOWS
return '\\';
#else
return '/';
#endif
}
// Returns the non-native path separator character for this platform.
inline char OtherPathSeparator() {
#ifdef PLATFORM_WINDOWS
return '/';
#else
return '\\';
#endif
}
// Closes the given FILE pointer once the object is deleted.
class FileCloser {
public:
explicit FileCloser(FILE* f) : fp_(f) {}
~FileCloser() {
if (fp_) fclose(fp_);
}
private:
FILE* fp_;
};
// All file paths are assumed to be UTF-8 encoded.
// Returns the |dir| that contains the current executable.
// The returned string does not have a trailing path separator.
absl::Status GetExeDir(std::string* dir);
// Returns a directory suitable for temporary files. The path is guaranteed to
// exist and to be a directory.
std::string GetTempDir();
#if PLATFORM_WINDOWS
enum class FolderId {
// E.g. C:\Users\jdoe\AppData\Roaming.
kRoamingAppData,
// E.g. C:\Program Files.
kProgramFiles
};
// Returns the Windows known folder path for the given |folder_id|.
absl::Status GetKnownFolderPath(FolderId folder_id, std::string* path);
// Expands environment path variables like %APPDATA%. Variables are matched
// case invariantly. Unknown environment variables are not changed.
absl::Status ExpandEnvironmentPathVariables(std::string* path);
#endif
// Returns the environment variable with given |name| in |value|.
// Returns a NotFound error and sets |value| to an empty string if the variable
// does not exist.
absl::Status GetEnv(const std::string& name, std::string* value);
// Sets the environment variable with given |name| to |value|. Only affects the
// current process, not system environment variables nor environment variables
// of other processes.
absl::Status SetEnv(const std::string& name, const std::string& value);
#if PLATFORM_WINDOWS
// Returns the part before the actual directory path without trailing path
// separator, i.e.
// - the drive letter "C:" for "C:\path\to\file" or
// - the network share "\\computer\share" for "\\computer\share\path\to\file".
// Returns an empty string if there is no such prefix (e.g. relative paths).
std::string GetDrivePrefix(const std::string& path);
#endif
// Gets the current working directory.
std::string GetCwd();
// Expands a relative path to an absolute path (relative to the current working
// directory). Also canonicalizes the path, removing any . and .. elements.
// Note that if the path does not exist or contains invalid characters, it may
// only be partially canonicalized, see
// https://en.cppreference.com/w/cpp/filesystem/canonical.
std::string GetFullPath(const std::string& path);
// Returns true if the given |path| is an absolute path.
bool IsAbsolute(const std::string& path);
// Returns the parent directory part of a |path|, without the trailing path
// separator. For instance, for "foo/bar", "foo/bar/", "foo\bar" or "foo\bar\",
// "foo" is returned.
// On Windows, if |path| only consists of a drive prefix, then the drive prefix
// is returned (see GetDrivePrefix()). If the path only has one component "foo",
// an empty string is returned.
std::string DirName(const std::string& path);
// Returns the last component of |path|. For instance, for "foo/bar",
// "foo/bar/", or "foo\bar", only "bar" is returned.
// If the path only has one component "bar", that component is returned.
std::string BaseName(const std::string& path);
// Joins |path| and |to_append|, adding a path separator if necessary.
std::string Join(absl::string_view path, absl::string_view to_append);
// Joins |path| and |to_append*|, adding a path separator if necessary.
std::string Join(absl::string_view path, absl::string_view to_append1,
absl::string_view to_append2);
// Joins |path| and |to_append*|, adding a path separator if necessary.
std::string Join(absl::string_view path, absl::string_view to_append1,
absl::string_view to_append2, absl::string_view to_append3);
// Joins |path| and |to_append|, adding a path separator if necessary, and
// stores the result in |dest|. |dest| must not overlap with |path| or
// |to_append|.
void Join(std::string* dest, absl::string_view path,
absl::string_view to_append);
// Joins |path| and |to_append|, adding a Unix path separator if necessary.
std::string JoinUnix(absl::string_view path, absl::string_view to_append);
// Appends |path| to |dest|, adding a path separator if necessary.
void Append(std::string* dest, absl::string_view to_append);
// Appends |path| to |dest|, adding a Unix path separator if necessary.
void AppendUnix(std::string* dest, absl::string_view to_append);
// Returns true if |path| ends with '\' or '//'.
bool EndsWithPathSeparator(const std::string& path);
// Adds a path separator at the end if there is none yet.
void EnsureEndsWithPathSeparator(std::string* path);
// Removes all path separators at the end if there are any.
void EnsureDoesNotEndWithPathSeparator(std::string* path);
// Converts path separators to the current platform's version, e.g. \ to /
// on Linux.
void FixPathSeparators(std::string* path);
// Converts a Windows path having backslashes as directory separators to a Unix
// path with forward slashes. A leading Windows drive letter like "C:" will be
// unchanged. For example: "C:\foo\bar" is converted to "C:/foo/bar".
std::string ToUnix(std::string path);
// Converts a path (eg, Windows or Unix path) to current system path.
// Substitutes forward slashes with backward slashes for Windows,
// substitutes backward slashes with forward slashes for Unix.
// A leading Windows drive letter like "C:" will be unchanged.
std::string ToNative(std::string path);
// Opens a file for the given |mode|.
absl::StatusOr<FILE*> OpenFile(const std::string& path, const char* mode);
using SearchHandler = std::function<absl::Status(
const std::string& dir, const std::string& filename, int64_t modified_time,
uint64_t size, bool is_dir)>;
// Searches all files and directories for which the path matches |pattern|.
//
// On Windows, if |pattern| is a directory path, finds that directory,
// traverses it, and finds all files and subdirectories in that directory.
// If |pattern| is a file path, finds that file. If |pattern| is
// a directory plus a file name pattern, e.g. "C:\files\*.t?t", returns all
// files and directories in that directory for which the path names match
// the pattern. Directory wildcards are not supported. A directory ending with
// "\" is treated like "\*", so it will not find the base directory, only
// subdirectories.
//
// On Linux, |pattern| must be a directory name and the function returns all
// files and directories in that directory. Glob-style patterns or file name
// matchings are not supported.
//
// If |recursive| is true, recurses into subdirectories. Does best effort,
// errors accessing directories does not stop the search, but prints out
// warnings (for access denied) or errors.
//
// Calls |handler| for every file and directory found. If the |handler| returns
// an error, the search is interrupted and the method returns that error.
// |modified_time| passed to |handler| is UTC time.
absl::Status SearchFiles(const std::string& pattern, bool recursive,
SearchHandler handler);
using StreamReadFileHandler =
std::function<absl::Status(const void* data, size_t data_size)>;
// Reads |file| in chunks of size |buffer_size| and calls |handler|. On EOF,
// calls handler(nullptr, 0). If |handler| returns an error, reading stops
// and the function returns that error. Also returns an error if a read fails.
absl::Status StreamReadFileContents(FILE* file, size_t buffer_size,
StreamReadFileHandler handler);
// Same as above, but uses the pre-allocated memory in |buffer|.
absl::Status StreamReadFileContents(FILE* file, Buffer* buffer,
StreamReadFileHandler handler);
using StreamWriteFileHandler =
std::function<absl::Status(const void** data, size_t* data_size)>;
// Calls |handler| and writes the data it returns to |file|. If |handler| sets
// (data, size) to (nullptr, 0) and returns ok, the function returns ok. If
// |handler| returns an error, the function returns that error. Also returns
// an error if a write fails.
absl::Status StreamWriteFileContents(FILE* file,
StreamWriteFileHandler handler);
// Reads the contents of the file at |path| to |data|.
absl::Status ReadFile(const std::string& path, Buffer* data);
// Reads at most |len| bytes starting at |offset| of the file
// at |path| into |data|. Returns the number of read bytes.
absl::StatusOr<size_t> ReadFile(const std::string& path, void* data,
size_t offset, size_t len);
// Reads the contents of the file at |path| and returns it as string.
absl::StatusOr<std::string> ReadFile(const std::string& path);
enum class ReadFlags {
kNone = 0, // Read all lines.
kTrimWhitespace = 1 << 0, // Trim whitespace from each line.
kRemoveEmpty = 1 << 1 // Remove empty lines (after optional trim).
};
// Reads the contents of the file at |path| line-by-line to |lines|.
absl::Status ReadAllLines(const std::string& path,
std::vector<std::string>* lines, ReadFlags flags);
// Writes |data| of |len| bytes to the file at |path|. If the file does not
// exist, it is created. Otherwise, its content is discarded.
absl::Status WriteFile(const std::string& path, const void* data, size_t len);
// Writes |data| to the file at |path|.
absl::Status WriteFile(const std::string& path, const Buffer& data);
// Writes |data| to the file at |path|.
absl::Status WriteFile(const std::string& path, const std::string& data);
// Creates a symlink at |link_path| pointing to |target|.
// |is_dir| should be true if |target| is a directory, otherwise false.
absl::Status CreateSymlink(const std::string& target,
const std::string& link_path, bool is_dir);
// Retrieves a symlink target at |link_path|.
absl::StatusOr<std::string> GetSymlinkTarget(const std::string& link_path);
// Checks the directory at |path| exists.
bool DirExists(const std::string& path);
// Checks the file or directory at |path| exists.
bool FileExists(const std::string& path);
// Checks the file or directory at |path| exists.
bool Exists(const std::string& path);
// Creates a directory for |path|.
// If the path already exists, the call is a no-op and returns success.
// Only creates the top-level directory. The call fails if intermediate
// directories are missing.
absl::Status CreateDir(const std::string& path);
// Creates a directory for |path|.
// If the path already exists, the call is a no-op and returns success.
// Also creates missing intermediate directories.
absl::Status CreateDirRec(const std::string& path);
// Renames/moves from |from_path| to |to_path|. Note that Windows fails when
// |to_path| exists, while Linux atomically replaces the |to_path| with
// |from_path|.
absl::Status RenameFile(const std::string& from_path,
const std::string& to_path);
// Copies from |from_path| to |to_path|.
// Also creates missing intermediate directories.
// Note that on Windows it's impossible to move or replace items between drives.
// In that case this method should be used.
absl::Status CopyFileRec(const std::string& from_path,
const std::string& to_path);
// Deletes the file at |path|. Returns OK if the file was deleted, or if
// |path| does not exist. On Windows, this requires write permissions to the
// file, while on Linux, it requires write permissions to the directory that
// contains the file.
absl::Status RemoveFile(const std::string& path);
// Deletes the file at |path| and moves the file at |replacement_path| to
// |old_path|.
absl::Status ReplaceFile(const std::string& path,
const std::string& replacement_path);
// Mode bits for GetStats. It's a bit unclear how these work on Windows.
// Apparently, the group/other flags are mirrored from the user flags.
enum Mode {
MODE_IFMT = 0170000, // File type bits
MODE_IFREG = 0100000, // Regular file
MODE_IFDIR = 0040000, // Directory
MODE_IFCHR = 0020000, // Character-oriented device file
MODE_IFIFO = 0010000, // FIFO or Pipe
MODE_ISUID = 0004000, // Set-user-ID on execute bit
MODE_ISGID = 0002000, // Set-group-ID on execute bit
MODE_ISVTX = 0001000, // Sticky bit
MODE_IRWXU = 0000700, // User rwx
MODE_IRUSR = 0000400, // User r
MODE_IWUSR = 0000200, // User w
MODE_IXUSR = 0000100, // User x
MODE_IRWXG = 0000070, // Group rwx
MODE_IRGRP = 0000040, // Group r
MODE_IWGRP = 0000020, // Group w
MODE_IXGRP = 0000010, // Group x
MODE_IRWXO = 0000007, // Other rwx
MODE_IROTH = 0000004, // Other r
MODE_IWOTH = 0000002, // Other w
MODE_IXOTH = 0000001, // Other x
};
struct Stats {
uint16_t mode = 0; // Mode enum
int64_t modified_time = 0; // Unix epoch (local time)
uint64_t size = 0;
};
// Returns stats for |path|. |modified_time| in
// the returned struct is local time. In other words
// it is the number of ticks from start of epoch by local time.
// TODO: Use consistent timestamps.
absl::Status GetStats(const std::string& path, Stats* stats);
// Returns the file size of the given file.
absl::Status FileSize(const std::string& path, uint64_t* size);
// Changes the mode bits of the file at |path| to |mode|.
absl::Status ChangeMode(const std::string& path, uint16_t mode);
// Removes |path| recursively.
absl::Status RemoveDirRec(const std::string& path);
// Reads the modification time of |path| and writes it to the given Unix
// epoch |timestamp| in UTC. |path| can point to a file or a directory.
absl::Status GetFileTime(const std::string& path, time_t* timestamp);
// Updates both the access and modification time of |path| to the given Unix
// epoch |timestamp| in UTC.
absl::Status SetFileTime(const std::string& path, time_t timestamp);
// Compares |path1| and |path2| and returns true if they are equal.
// Converts both paths to canonical form before coparison.
bool AreEqual(std::string path1, std::string path2);
} // namespace path
inline path::ReadFlags operator|(path::ReadFlags a, path::ReadFlags b) {
using T = std::underlying_type_t<path::ReadFlags>;
return static_cast<path::ReadFlags>(static_cast<T>(a) | static_cast<T>(b));
}
inline path::ReadFlags operator&(path::ReadFlags a, path::ReadFlags b) {
using T = std::underlying_type_t<path::ReadFlags>;
return static_cast<path::ReadFlags>(static_cast<T>(a) & static_cast<T>(b));
}
} // namespace cdc_ft
#endif // COMMON_PATH_H_

80
common/path_filter.cc Normal file
View File

@@ -0,0 +1,80 @@
// Copyright 2022 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "common/path_filter.h"
namespace cdc_ft {
namespace internal {
bool IsMatch(const char* pattern, const char* path, size_t path_len) {
for (/*empty*/; *pattern != '\0'; ++pattern) {
switch (*pattern) {
case '?':
if (*path == '\0') {
return false;
}
++path;
break;
case '*': {
if (pattern[1] == '\0') {
return true;
}
for (size_t n = 0; n < path_len; n++) {
if (IsMatch(pattern + 1, path + n, path_len - n)) {
return true;
}
}
return false;
}
default:
if (*path != *pattern) {
return false;
}
++path;
}
}
return *path == '\0';
}
bool IsMatch(const std::string& pattern, const std::string& path) {
return IsMatch(pattern.c_str(), path.c_str(), path.size());
}
} // namespace internal
PathFilter::PathFilter() = default;
PathFilter::~PathFilter() = default;
bool PathFilter::IsEmpty() const { return rules_.empty(); }
void PathFilter::AddRule(Rule::Type type, std::string pattern) {
rules_.emplace_back(type, std::move(pattern));
}
bool PathFilter::IsMatch(const std::string& path) const {
for (const Rule& rule : rules_) {
if (internal::IsMatch(rule.pattern.c_str(), path.c_str(), path.size())) {
return rule.type == Rule::Type::kInclude;
}
}
return true;
}
PathFilter::Rule::Rule(Type type, std::string pattern)
: type(type), pattern(std::move(pattern)) {}
} // namespace cdc_ft

72
common/path_filter.h Normal file
View File

@@ -0,0 +1,72 @@
/*
* Copyright 2022 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef COMMON_PATH_FILTER_H_
#define COMMON_PATH_FILTER_H_
#include <string>
#include <vector>
namespace cdc_ft {
namespace internal {
// Returns true iff |path| matches |pattern|. Exposed for testing purposes.
bool IsMatch(const std::string& pattern, const std::string& path);
} // namespace internal
// Applies include and exclude filter rules to a path.
class PathFilter {
public:
struct Rule {
enum class Type {
kInclude,
kExclude,
};
Type type;
std::string pattern;
Rule(Type type, std::string pattern);
};
PathFilter();
~PathFilter();
// Returns true if no rule has been added.
bool IsEmpty() const;
// Adds a rule to the set of filter rules. The |type| determines whether paths
// that match |pattern| should be filtered in or out, see IsMatch(). |pattern|
// is a Windows style file path pattern and supports * and ? wildcards. Does
// not support general glob-style ** or [a-z] patterns.
void AddRule(Rule::Type type, std::string pattern);
// Returns all rules added.
const std::vector<Rule>& GetRules() const { return rules_; }
// Applies filter rules in the order they have been passed to AddRule().
// Returns true if the first matching rule is of type |kInclude| or no
// matching rule is found. Returns false if the first matching rule is
// of type |kExclude|.
bool IsMatch(const std::string& path) const;
private:
std::vector<Rule> rules_;
};
} // namespace cdc_ft
#endif // COMMON_PATH_FILTER_H_

130
common/path_filter_test.cc Normal file
View File

@@ -0,0 +1,130 @@
// Copyright 2022 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "common/path_filter.h"
#include "gtest/gtest.h"
namespace cdc_ft {
namespace {
class PathFilterTest : public ::testing::Test {};
TEST_F(PathFilterTest, IsMatchFn_NoWildcard) {
EXPECT_TRUE(internal::IsMatch("", ""));
EXPECT_TRUE(internal::IsMatch("a", "a"));
EXPECT_FALSE(internal::IsMatch("a", "b"));
EXPECT_TRUE(internal::IsMatch("abc", "abc"));
EXPECT_FALSE(internal::IsMatch("abc", "abd"));
}
TEST_F(PathFilterTest, IsMatchFn_Questionmark) {
EXPECT_FALSE(internal::IsMatch("?", ""));
EXPECT_TRUE(internal::IsMatch("?", "a"));
EXPECT_FALSE(internal::IsMatch("?", "ab"));
EXPECT_TRUE(internal::IsMatch("??", "ab"));
EXPECT_FALSE(internal::IsMatch("??", "a"));
}
TEST_F(PathFilterTest, IsMatchFn_Asterisk) {
EXPECT_TRUE(internal::IsMatch("*", ""));
EXPECT_TRUE(internal::IsMatch("*", "a"));
EXPECT_TRUE(internal::IsMatch("*", "ab"));
EXPECT_TRUE(internal::IsMatch("*", "abc"));
EXPECT_TRUE(internal::IsMatch("a**", "abc"));
EXPECT_FALSE(internal::IsMatch("b**", "abc"));
EXPECT_FALSE(internal::IsMatch("c**", "abc"));
EXPECT_TRUE(internal::IsMatch("*a*", "abc"));
EXPECT_TRUE(internal::IsMatch("*b*", "abc"));
EXPECT_TRUE(internal::IsMatch("*c*", "abc"));
EXPECT_FALSE(internal::IsMatch("**a", "abc"));
EXPECT_FALSE(internal::IsMatch("**b", "abc"));
EXPECT_TRUE(internal::IsMatch("**c", "abc"));
}
TEST_F(PathFilterTest, IsMatchFn_CommonCases) {
constexpr char p1[] = "*.txt";
EXPECT_TRUE(internal::IsMatch(p1, "file1.txt"));
EXPECT_FALSE(internal::IsMatch(p1, "file2.dat"));
constexpr char p2[] = "dir1\\dir2\\*.txt";
EXPECT_TRUE(internal::IsMatch(p2, "dir1\\dir2\\file1.txt"));
EXPECT_FALSE(internal::IsMatch(p2, "dir1\\dir2\\file2.dat"));
EXPECT_FALSE(internal::IsMatch(p2, "dir1\\file3.txt"));
constexpr char p3[] = "dir1\\dir?\\*.txt";
EXPECT_TRUE(internal::IsMatch(p3, "dir1\\dir2\\file1.txt"));
EXPECT_TRUE(internal::IsMatch(p3, "dir1\\dir3\\file2.txt"));
EXPECT_FALSE(internal::IsMatch(p3, "dir1\\file3.txt"));
constexpr char p4[] = "dir1\\*\\*.txt";
EXPECT_FALSE(internal::IsMatch(p4, "dir1\\file1.txt"));
EXPECT_TRUE(internal::IsMatch(p4, "dir1\\dir2\\file1.txt"));
EXPECT_TRUE(internal::IsMatch(p4, "dir1\\dir3\\file2.txt"));
// Note: This is different from glob behavior!
EXPECT_TRUE(internal::IsMatch(p4, "dir1\\dir4\\dir5\\file3.txt"));
EXPECT_TRUE(internal::IsMatch(p4, "dir1\\\\file2.txt"));
constexpr char p5[] = "dir1\\*\\dir3\\*\\*.txt";
EXPECT_FALSE(internal::IsMatch(p5, "dir1\\dir2\\file1.txt"));
EXPECT_FALSE(internal::IsMatch(p5, "dir1\\dir3\\file2.txt"));
EXPECT_TRUE(internal::IsMatch(p5, "dir1\\dir2\\dir3\\dir4\\file3.txt"));
}
TEST_F(PathFilterTest, IsMatch_NoFilter) {
PathFilter path_filter;
EXPECT_TRUE(path_filter.IsMatch(""));
EXPECT_TRUE(path_filter.IsMatch("a"));
EXPECT_TRUE(path_filter.IsMatch("b\\c"));
}
TEST_F(PathFilterTest, IsMatch_IncludeFilter) {
PathFilter path_filter;
path_filter.AddRule(PathFilter::Rule::Type::kInclude, "a");
EXPECT_TRUE(path_filter.IsMatch(""));
EXPECT_TRUE(path_filter.IsMatch("a"));
EXPECT_TRUE(path_filter.IsMatch("b\\c"));
}
TEST_F(PathFilterTest, IsMatch_ExcludeFilter) {
PathFilter path_filter;
path_filter.AddRule(PathFilter::Rule::Type::kExclude, "a");
EXPECT_TRUE(path_filter.IsMatch(""));
EXPECT_FALSE(path_filter.IsMatch("a"));
EXPECT_TRUE(path_filter.IsMatch("b\\c"));
}
TEST_F(PathFilterTest, IsMatch_IncludeThenExclude) {
PathFilter path_filter;
path_filter.AddRule(PathFilter::Rule::Type::kInclude, "file2.txt");
path_filter.AddRule(PathFilter::Rule::Type::kExclude, "*.txt");
EXPECT_FALSE(path_filter.IsMatch("file1.txt"));
EXPECT_TRUE(path_filter.IsMatch("file2.txt"));
EXPECT_TRUE(path_filter.IsMatch("file3.dat"));
}
TEST_F(PathFilterTest, IsMatch_ExcludeThenInclude) {
PathFilter path_filter;
path_filter.AddRule(PathFilter::Rule::Type::kExclude, "*.txt");
path_filter.AddRule(PathFilter::Rule::Type::kInclude, "file2.txt");
EXPECT_FALSE(path_filter.IsMatch("file1.txt"));
EXPECT_FALSE(path_filter.IsMatch("file2.txt"));
EXPECT_TRUE(path_filter.IsMatch("file3.dat"));
}
} // namespace
} // namespace cdc_ft

1521
common/path_test.cc Normal file
View File

File diff suppressed because it is too large Load Diff

30
common/platform.h Normal file
View File

@@ -0,0 +1,30 @@
/*
* Copyright 2022 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef COMMON_PLATFORM_H_
#define COMMON_PLATFORM_H_
#undef PLATFORM_LINUX
#undef PLATFORM_WINDOWS
#ifdef __linux__
#define PLATFORM_LINUX 1
#define PLATFORM_NAME "Linux"
#elif _WIN32
#define PLATFORM_WINDOWS 1
#define PLATFORM_NAME "Windows"
#endif
#endif // COMMON_PLATFORM_H_

113
common/port_manager.h Normal file
View File

@@ -0,0 +1,113 @@
/*
* Copyright 2022 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef COMMON_PORT_MANAGER_H_
#define COMMON_PORT_MANAGER_H_
#include <absl/status/statusor.h>
#include <memory>
#include <string>
#include <unordered_set>
#include "common/clock.h"
namespace cdc_ft {
class ProcessFactory;
class RemoteUtil;
class SharedMemory;
// Class for reserving ports globally. Use if there can be multiple processes
// of the same type that might request ports at the same time, e.g. multiple
// cdc_rsync.exe processes running concurrently.
class PortManager {
public:
// |unique_name| is a globally unique name used for shared memory to
// synchronize port reservation. The range of possible ports managed by this
// instance is [|first_port|, |last_port|]. |process_factory| is a valid
// pointer to a ProcessFactory instance to run processes locally.
// |remote_util| is a valid pointer to a RemoteUtil instance to run processes
// remotely.
PortManager(std::string unique_name, int first_port, int last_port,
ProcessFactory* process_factory, RemoteUtil* remote_util,
SystemClock* system_clock = DefaultSystemClock::GetInstance(),
SteadyClock* steady_clock = DefaultSteadyClock::GetInstance());
~PortManager();
// Reserves a port in the range passed to the constructor. The port is
// released automatically upon destruction if ReleasePort() is not called
// explicitly.
// |timeout_sec| is the timeout for finding available ports on the gamelet
// instance. Returns a DeadlineExceeded error if the timeout is exceeded.
// Returns a ResourceExhausted error if no ports are available.
absl::StatusOr<int> ReservePort(int timeout_sec);
// Releases a reserved port.
absl::Status ReleasePort(int port);
//
// Lower-level interface for finding available ports directly.
//
// Finds available ports in the range [first_port, last_port] for port
// forwarding on the local workstation.
// |ip| is the IP address to filter by.
// |process_factory| is used to create a netstat process.
// |forward_output_to_log| determines whether the stderr of netstat is
// forwarded to the logs. Returns ResourceExhaustedError if no port is
// available.
static absl::StatusOr<std::unordered_set<int>> FindAvailableLocalPorts(
int first_port, int last_port, const char* ip,
ProcessFactory* process_factory, bool forward_output_to_log);
// Finds available ports in the range [first_port, last_port] for port
// forwarding on the instance.
// |ip| is the IP address to filter by.
// |process_factory| is used to create a netstat process.
// |remote_util| is used to connect to the instance.
// |timeout_sec| is the connection timeout in seconds.
// |forward_output_to_log| determines whether the stderr of netstat is
// forwarded to the logs. Returns a DeadlineExceeded error if the timeout is
// exceeded. Returns ResourceExhaustedError if no port is available.
static absl::StatusOr<std::unordered_set<int>> FindAvailableRemotePorts(
int first_port, int last_port, const char* ip,
ProcessFactory* process_factory, RemoteUtil* remote_util, int timeout_sec,
bool forward_output_to_log,
SteadyClock* steady_clock = DefaultSteadyClock::GetInstance());
private:
// Returns a list of available ports in the range [|first_port|, |last_port|]
// from the given |netstat_output|. |ip| is the IP address to look for, e.g.
// "127.0.0.1".
// Returns ResourceExhaustedError if no port is available.
static absl::StatusOr<std::unordered_set<int>> FindAvailablePorts(
int first_port, int last_port, const std::string& netstat_output,
const char* ip);
int first_port_;
int last_port_;
ProcessFactory* process_factory_;
RemoteUtil* remote_util_;
SystemClock* system_clock_;
SteadyClock* steady_clock_;
std::unique_ptr<SharedMemory> shared_mem_;
std::unordered_set<int> reserved_ports_;
};
} // namespace cdc_ft
#endif // COMMON_PORT_MANAGER_H_

256
common/port_manager_test.cc Normal file
View File

@@ -0,0 +1,256 @@
// Copyright 2022 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "common/port_manager.h"
#include "absl/strings/match.h"
#include "common/log.h"
#include "common/remote_util.h"
#include "common/status_test_macros.h"
#include "common/stub_process.h"
#include "common/testing_clock.h"
#include "gtest/gtest.h"
namespace cdc_ft {
namespace {
constexpr int kGameletPort = 12345;
constexpr char kGameletIp[] = "1.2.3.4";
constexpr char kGuid[] = "f77bcdfe-368c-4c45-9f01-230c5e7e2132";
constexpr int kFirstPort = 44450;
constexpr int kLastPort = 44459;
constexpr int kNumPorts = kLastPort - kFirstPort + 1;
constexpr int kTimeoutSec = 1;
constexpr char kLocalNetstat[] = "netstat -a -n -p tcp";
constexpr char kRemoteNetstat[] = "netstat --numeric --listening --tcp";
constexpr char kLocalNetstatOutFmt[] =
"TCP 127.0.0.1:50000 127.0.0.1:%i ESTABLISHED";
constexpr char kRemoteNetstatOutFmt[] =
"tcp 0 0 0.0.0.0:%i 0.0.0.0:* LISTEN";
class PortManagerTest : public ::testing::Test {
public:
PortManagerTest()
: remote_util_(/*verbosity=*/0, /*quiet=*/false, &process_factory_,
/*forward_output_to_log=*/true),
port_manager_(kGuid, kFirstPort, kLastPort, &process_factory_,
&remote_util_, &system_clock_, &steady_clock_) {}
void SetUp() override {
Log::Initialize(std::make_unique<ConsoleLog>(LogLevel::kInfo));
remote_util_.SetIpAndPort(kGameletIp, kGameletPort);
}
void TearDown() override { Log::Shutdown(); }
protected:
StubProcessFactory process_factory_;
TestingSystemClock system_clock_;
TestingSteadyClock steady_clock_;
RemoteUtil remote_util_;
PortManager port_manager_;
};
TEST_F(PortManagerTest, ReservePortSuccess) {
process_factory_.SetProcessOutput(kLocalNetstat, "", "", 0);
process_factory_.SetProcessOutput(kRemoteNetstat, "", "", 0);
absl::StatusOr<int> port = port_manager_.ReservePort(kTimeoutSec);
ASSERT_OK(port);
EXPECT_EQ(*port, kFirstPort);
}
TEST_F(PortManagerTest, ReservePortAllLocalPortsTaken) {
std::string local_netstat_out = "";
for (int port = kFirstPort; port <= kLastPort; ++port) {
local_netstat_out += absl::StrFormat(kLocalNetstatOutFmt, port);
}
process_factory_.SetProcessOutput(kLocalNetstat, local_netstat_out, "", 0);
process_factory_.SetProcessOutput(kRemoteNetstat, "", "", 0);
absl::StatusOr<int> port = port_manager_.ReservePort(kTimeoutSec);
EXPECT_TRUE(absl::IsResourceExhausted(port.status()));
EXPECT_TRUE(
absl::StrContains(port.status().message(), "No port available in range"));
}
TEST_F(PortManagerTest, ReservePortAllRemotePortsTaken) {
std::string remote_netstat_out = "";
for (int port = kFirstPort; port <= kLastPort; ++port) {
remote_netstat_out += absl::StrFormat(kRemoteNetstatOutFmt, port);
}
process_factory_.SetProcessOutput(kLocalNetstat, "", "", 0);
process_factory_.SetProcessOutput(kRemoteNetstat, remote_netstat_out, "", 0);
absl::StatusOr<int> port = port_manager_.ReservePort(kTimeoutSec);
EXPECT_TRUE(absl::IsResourceExhausted(port.status()));
EXPECT_TRUE(
absl::StrContains(port.status().message(), "No port available in range"));
}
TEST_F(PortManagerTest, ReservePortLocalNetstatFails) {
process_factory_.SetProcessOutput(kLocalNetstat, "", "", 1);
process_factory_.SetProcessOutput(kRemoteNetstat, "", "", 0);
absl::StatusOr<int> port = port_manager_.ReservePort(kTimeoutSec);
EXPECT_NOT_OK(port);
EXPECT_TRUE(
absl::StrContains(port.status().message(),
"Failed to find available ports on workstation"));
}
TEST_F(PortManagerTest, ReservePortRemoteNetstatFails) {
process_factory_.SetProcessOutput(kLocalNetstat, "", "", 0);
process_factory_.SetProcessOutput(kRemoteNetstat, "", "", 1);
absl::StatusOr<int> port = port_manager_.ReservePort(kTimeoutSec);
EXPECT_NOT_OK(port);
EXPECT_TRUE(absl::StrContains(port.status().message(),
"Failed to find available ports on instance"));
}
TEST_F(PortManagerTest, ReservePortRemoteNetstatTimesOut) {
process_factory_.SetProcessOutput(kLocalNetstat, "", "", 0);
process_factory_.SetProcessNeverExits(kRemoteNetstat);
steady_clock_.AutoAdvance(kTimeoutSec * 2 * 1000);
absl::StatusOr<int> port = port_manager_.ReservePort(kTimeoutSec);
EXPECT_NOT_OK(port);
EXPECT_TRUE(absl::IsDeadlineExceeded(port.status()));
EXPECT_TRUE(absl::StrContains(port.status().message(),
"Timeout while running netstat"));
}
TEST_F(PortManagerTest, ReservePortMultipleInstances) {
process_factory_.SetProcessOutput(kLocalNetstat, "", "", 0);
process_factory_.SetProcessOutput(kRemoteNetstat, "", "", 0);
PortManager port_manager2(kGuid, kFirstPort, kLastPort, &process_factory_,
&remote_util_);
// Port managers use shared memory, so different instances know about each
// other. This would even work if |port_manager_| and |port_manager2| belonged
// to different processes, but we don't test that here.
EXPECT_EQ(*port_manager_.ReservePort(kTimeoutSec), kFirstPort + 0);
EXPECT_EQ(*port_manager2.ReservePort(kTimeoutSec), kFirstPort + 1);
EXPECT_EQ(*port_manager_.ReservePort(kTimeoutSec), kFirstPort + 2);
EXPECT_EQ(*port_manager2.ReservePort(kTimeoutSec), kFirstPort + 3);
}
TEST_F(PortManagerTest, ReservePortReusesPortsInLRUOrder) {
process_factory_.SetProcessOutput(kLocalNetstat, "", "", 0);
process_factory_.SetProcessOutput(kRemoteNetstat, "", "", 0);
for (int n = 0; n < kNumPorts * 2; ++n) {
EXPECT_EQ(*port_manager_.ReservePort(kTimeoutSec),
kFirstPort + n % kNumPorts);
system_clock_.Advance(1000);
}
}
TEST_F(PortManagerTest, ReleasePort) {
process_factory_.SetProcessOutput(kLocalNetstat, "", "", 0);
process_factory_.SetProcessOutput(kRemoteNetstat, "", "", 0);
absl::StatusOr<int> port = port_manager_.ReservePort(kTimeoutSec);
EXPECT_EQ(*port, kFirstPort);
EXPECT_OK(port_manager_.ReleasePort(*port));
port = port_manager_.ReservePort(kTimeoutSec);
EXPECT_EQ(*port, kFirstPort);
}
TEST_F(PortManagerTest, ReleasePortOnDestruction) {
process_factory_.SetProcessOutput(kLocalNetstat, "", "", 0);
process_factory_.SetProcessOutput(kRemoteNetstat, "", "", 0);
auto port_manager2 = std::make_unique<PortManager>(
kGuid, kFirstPort, kLastPort, &process_factory_, &remote_util_);
EXPECT_EQ(*port_manager2->ReservePort(kTimeoutSec), kFirstPort + 0);
EXPECT_EQ(*port_manager_.ReservePort(kTimeoutSec), kFirstPort + 1);
port_manager2.reset();
EXPECT_EQ(*port_manager_.ReservePort(kTimeoutSec), kFirstPort + 0);
}
TEST_F(PortManagerTest, FindAvailableLocalPortsSuccess) {
// First port is taken
std::string local_netstat_out =
absl::StrFormat(kLocalNetstatOutFmt, kFirstPort);
process_factory_.SetProcessOutput(kLocalNetstat, local_netstat_out, "", 0);
absl::StatusOr<std::unordered_set<int>> ports =
PortManager::FindAvailableLocalPorts(kFirstPort, kLastPort, "127.0.0.1",
&process_factory_, true);
ASSERT_OK(ports);
EXPECT_EQ(ports->size(), kNumPorts - 1);
for (int port = kFirstPort + 1; port <= kLastPort; ++port) {
EXPECT_TRUE(ports->find(port) != ports->end());
}
}
TEST_F(PortManagerTest, FindAvailableLocalPortsFailsNoPorts) {
// All ports taken
std::string local_netstat_out = "";
for (int port = kFirstPort; port <= kLastPort; ++port) {
local_netstat_out += absl::StrFormat(kLocalNetstatOutFmt, port);
}
process_factory_.SetProcessOutput(kLocalNetstat, local_netstat_out, "", 0);
absl::StatusOr<std::unordered_set<int>> ports =
PortManager::FindAvailableLocalPorts(kFirstPort, kLastPort, "127.0.0.1",
&process_factory_, true);
EXPECT_TRUE(absl::IsResourceExhausted(ports.status()));
EXPECT_TRUE(absl::StrContains(ports.status().message(),
"No port available in range"));
}
TEST_F(PortManagerTest, FindAvailableRemotePortsSuccess) {
// First port is taken
std::string remote_netstat_out =
absl::StrFormat(kRemoteNetstatOutFmt, kFirstPort);
process_factory_.SetProcessOutput(kRemoteNetstat, remote_netstat_out, "", 0);
absl::StatusOr<std::unordered_set<int>> ports =
PortManager::FindAvailableRemotePorts(kFirstPort, kLastPort, "0.0.0.0",
&process_factory_, &remote_util_,
kTimeoutSec, true);
ASSERT_OK(ports);
EXPECT_EQ(ports->size(), kNumPorts - 1);
for (int port = kFirstPort + 1; port <= kLastPort; ++port) {
EXPECT_TRUE(ports->find(port) != ports->end());
}
}
TEST_F(PortManagerTest, FindAvailableRemotePortsFailsNoPorts) {
// All ports taken
std::string remote_netstat_out = "";
for (int port = kFirstPort; port <= kLastPort; ++port) {
remote_netstat_out += absl::StrFormat(kRemoteNetstatOutFmt, port);
}
process_factory_.SetProcessOutput(kRemoteNetstat, remote_netstat_out, "", 0);
absl::StatusOr<std::unordered_set<int>> ports =
PortManager::FindAvailableRemotePorts(kFirstPort, kLastPort, "0.0.0.0",
&process_factory_, &remote_util_,
kTimeoutSec, true);
EXPECT_TRUE(absl::IsResourceExhausted(ports.status()));
EXPECT_TRUE(absl::StrContains(ports.status().message(),
"No port available in range"));
}
} // namespace
} // namespace cdc_ft

302
common/port_manager_win.cc Normal file
View File

@@ -0,0 +1,302 @@
// Copyright 2022 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "common/port_manager.h"
#define WIN32_LEAN_AND_MEAN
#include <windows.h>
#include <map>
#include "absl/strings/str_split.h"
#include "common/log.h"
#include "common/process.h"
#include "common/remote_util.h"
#include "common/status.h"
#include "common/status_macros.h"
#include "common/stopwatch.h"
#include "common/util.h"
namespace cdc_ft {
class SharedMemory {
public:
// Creates a new shared memory instance with given |name| and |size| in bytes.
// Different instances with matching names reference the same piece of memory,
// even if they belong to different processes. If shared memory with the given
// |name| already exists, the existing memory is referenced. Otherwise, a new
// piece of memory is allocated and zero-initialized.
SharedMemory(std::string name, size_t size)
: name_(std::move(name)), size_(size) {}
absl::StatusOr<void*> Get() {
// Already initialized?
if (shared_mem_) return shared_mem_;
assert(!map_file_handle_);
LARGE_INTEGER size;
size.QuadPart = size_;
map_file_handle_ = CreateFileMapping(
INVALID_HANDLE_VALUE, // use paging file
nullptr, // default security
PAGE_READWRITE, // read/write access
size.HighPart, // maximum object size (high-order DWORD)
size.LowPart, // maximum object size (low-order DWORD)
Util::Utf8ToWideStr(name_).c_str()); // name of mapping object
if (!map_file_handle_) {
return MakeStatus("Failed to create file mapping object: %s",
Util::GetLastWin32Error());
}
// The shared memory holds the timestamps when the ports were reserved.
shared_mem_ = MapViewOfFile(map_file_handle_, // handle to map object
FILE_MAP_ALL_ACCESS, // read/write permission
0, 0, size.QuadPart);
if (!shared_mem_) {
std::string errorMessage = Util::GetLastWin32Error();
CloseHandle(map_file_handle_);
map_file_handle_ = nullptr;
return MakeStatus("Failed to map view of file: %s", errorMessage);
}
return shared_mem_;
}
~SharedMemory() {
if (shared_mem_) {
UnmapViewOfFile(shared_mem_);
shared_mem_ = nullptr;
}
if (map_file_handle_) {
CloseHandle(map_file_handle_);
map_file_handle_ = nullptr;
}
}
private:
std::string name_;
size_t size_;
HANDLE map_file_handle_ = nullptr;
void* shared_mem_ = nullptr;
};
PortManager::PortManager(std::string name, int first_port, int last_port,
ProcessFactory* process_factory,
RemoteUtil* remote_util, SystemClock* system_clock,
SteadyClock* steady_clock)
: first_port_(first_port),
last_port_(last_port),
process_factory_(process_factory),
remote_util_(remote_util),
system_clock_(system_clock),
steady_clock_(steady_clock),
shared_mem_(std::make_unique<SharedMemory>(
std::move(name), (last_port - first_port + 1) * sizeof(time_t))) {
assert(last_port_ >= first_port_);
}
PortManager::~PortManager() {
std::vector<int> ports_copy;
ports_copy.insert(ports_copy.end(), reserved_ports_.begin(),
reserved_ports_.end());
for (int port : ports_copy) {
absl::Status status = ReleasePort(port);
if (!status.ok()) {
LOG_WARNING("Failed to release port %d: %s", port, status.ToString());
}
}
}
absl::StatusOr<int> PortManager::ReservePort(int timeout_sec) {
// Find available port on workstation.
std::unordered_set<int> local_ports;
ASSIGN_OR_RETURN(local_ports,
FindAvailableLocalPorts(first_port_, last_port_, "127.0.0.1",
process_factory_, false),
"Failed to find available ports on workstation");
// Find available port on remote gamelet.
std::unordered_set<int> remote_ports;
ASSIGN_OR_RETURN(remote_ports,
FindAvailableRemotePorts(first_port_, last_port_, "0.0.0.0",
process_factory_, remote_util_,
timeout_sec, false, steady_clock_),
"Failed to find available ports on instance");
// Fetch shared memory.
void* mem;
ASSIGN_OR_RETURN(mem, shared_mem_->Get(), "Failed to get shared memory");
time_t* port_timestamps = static_cast<time_t*>(mem);
// Put ports into a multimap to iterate in LRU order.
int num_ports = last_port_ - first_port_ + 1;
std::multimap<time_t, int> ports_to_index;
for (int n = 0; n < num_ports; ++n) {
ports_to_index.insert({port_timestamps[n], n});
}
// Iterate over the ports, unused first (timestamp 0), the rest in LRU order.
// The ones with timestamps != 0 might either be stuck (e.g. process crashed
// and did not release port) or still in use.
const time_t now = std::chrono::system_clock::to_time_t(system_clock_->Now());
for (const auto& [port_timestamp, n] : ports_to_index) {
// Note that some other process might have hijacked the port in the
// meantime, hence do an InterlockedCompareExchange.
volatile time_t* ts_ptr = &port_timestamps[n];
static_assert(sizeof(time_t) == sizeof(uint64_t), "time_t must be 64 bit");
assert((reinterpret_cast<uintptr_t>(ts_ptr) & 7) == 0);
if (InterlockedCompareExchange64(ts_ptr, now, port_timestamp) ==
port_timestamp) {
int port = first_port_ + n;
LOG_DEBUG("Trying to reserve port %i", port);
// We have reserved this port. Double-check that it's actually not in use
// on both the workstation and the server.
if (local_ports.find(port) == local_ports.end()) {
LOG_DEBUG("Port %i not available on workstation", port);
InterlockedCompareExchange64(ts_ptr, now, port_timestamp);
continue;
}
if (remote_ports.find(port) == remote_ports.end()) {
LOG_DEBUG("Port %i not available on instance", port);
InterlockedCompareExchange64(ts_ptr, now, port_timestamp);
continue;
}
LOG_DEBUG("Port %i is available on workstation and instance", port);
reserved_ports_.insert(port);
return port;
}
}
return absl::ResourceExhaustedError(absl::StrFormat(
"No port available in range [%i, %i]", first_port_, last_port_));
}
absl::Status PortManager::ReleasePort(int port) {
if (reserved_ports_.find(port) == reserved_ports_.end())
return absl::OkStatus();
void* mem;
ASSIGN_OR_RETURN(mem, shared_mem_->Get(), "Failed to get shared memory");
time_t* port_timestamps = static_cast<time_t*>(mem);
volatile time_t* ts_ptr = &port_timestamps[port - first_port_];
InterlockedExchange64(ts_ptr, 0);
reserved_ports_.erase(port);
return absl::OkStatus();
}
// static
absl::StatusOr<std::unordered_set<int>> PortManager::FindAvailableLocalPorts(
int first_port, int last_port, const char* ip,
ProcessFactory* process_factory, bool forward_output_to_log) {
// -a to get the connection and ports the computer is listening on.
// -n to get numerical addresses to avoid the overhead of determining names.
// -p tcp to limit the output to TCPv4 connections.
// TODO: Use Windows API instead of netstat.
ProcessStartInfo start_info;
start_info.command = "netstat -a -n -p tcp";
start_info.name = "netstat";
std::string output;
start_info.stdout_handler = [&output](const char* data, size_t data_size) {
output.append(data, data_size);
return absl::OkStatus();
};
start_info.forward_output_to_log = forward_output_to_log;
absl::Status status = process_factory->Run(start_info);
if (!status.ok()) return WrapStatus(status, "Failed to run netstat");
LOG_DEBUG("netstat (workstation) output:\n%s", output);
return FindAvailablePorts(first_port, last_port, output, ip);
}
// static
absl::StatusOr<std::unordered_set<int>> PortManager::FindAvailableRemotePorts(
int first_port, int last_port, const char* ip,
ProcessFactory* process_factory, RemoteUtil* remote_util, int timeout_sec,
bool forward_output_to_log, SteadyClock* steady_clock) {
// --numeric to get numerical addresses.
// --listening to get only listening sockets.
// --tcp to get only TCP connections.
std::string remote_command = "netstat --numeric --listening --tcp";
ProcessStartInfo start_info =
remote_util->BuildProcessStartInfoForSsh(remote_command);
start_info.name = "netstat";
std::string output;
start_info.stdout_handler = [&output](const char* data, size_t data_size) {
output.append(data, data_size);
return absl::OkStatus();
};
start_info.forward_output_to_log = forward_output_to_log;
std::unique_ptr<Process> process = process_factory->Create(start_info);
absl::Status status = process->Start();
if (!status.ok())
return WrapStatus(status, "Failed to start netstat process");
Stopwatch timeout_timer(steady_clock);
bool is_timeout = false;
auto detect_timeout = [&timeout_timer, timeout_sec, &is_timeout]() {
is_timeout = timeout_timer.ElapsedSeconds() > timeout_sec;
return is_timeout;
};
status = process->RunUntil(detect_timeout);
if (!status.ok()) return WrapStatus(status, "Failed to run netstat process");
if (is_timeout)
return absl::DeadlineExceededError("Timeout while running netstat");
uint32_t exit_code = process->ExitCode();
if (exit_code != 0)
return MakeStatus("netstat process exited with code %u", exit_code);
LOG_DEBUG("netstat (instance) output:\n%s", output);
return FindAvailablePorts(first_port, last_port, output, ip);
}
// static
absl::StatusOr<std::unordered_set<int>> PortManager::FindAvailablePorts(
int first_port, int last_port, const std::string& netstat_output,
const char* ip) {
std::unordered_set<int> available_ports;
for (int port = first_port; port <= last_port; ++port) {
std::vector<std::string> lines = absl::StrSplit(netstat_output, '\n');
bool port_occupied = false;
std::string portToken = absl::StrFormat("%s:%i", ip, port);
for (const std::string& line : lines) {
// Ports in the TIME_WAIT state can be reused. It is common that ports
// stay in this state for O(minutes).
if (absl::StrContains(line, portToken) &&
!absl::StrContains(line, "TIME_WAIT")) {
port_occupied = true;
break;
}
}
if (!port_occupied) available_ports.insert(port);
}
if (available_ports.empty()) {
return absl::ResourceExhaustedError(absl::StrFormat(
"No port available in range [%i, %i]", first_port, last_port));
}
return available_ports;
}
} // namespace cdc_ft

145
common/process.h Normal file
View File

@@ -0,0 +1,145 @@
/*
* Copyright 2022 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef COMMON_PROCESS_H_
#define COMMON_PROCESS_H_
#include <functional>
#include <memory>
#include <string>
#include "absl/status/status.h"
#include "common/log.h"
namespace cdc_ft {
// Helper function to forward process output to the logs.
// |name| is prefixed to the logs.
// |log_level| specifies the log level. If not given, the level is guessed from
// the log lines.
absl::Status LogOutput(const char* name, const char* data, size_t data_size,
absl::optional<LogLevel> log_level = {});
struct ProcessStartInfo {
// Handler for stdout/stderr. |data| is guaranteed to be NULL terminated, so
// it may be used like a C-string if it's known to be text, e.g. for printf().
// The NULL terminator does not count towards |data_size|.
using OutputHandler =
std::function<absl::Status(const char* data, size_t data_size)>;
// Human readable name for debugging purposes, e.g. the message pump thread
// name. Falls back to command.
std::string name;
// Command line, UTF-8 encoded.
std::string command;
// If set, the process stdin is redirected to a pipe.
// It not set, the input is connected to the stdin of the calling process.
bool redirect_stdin = false;
// If set, forwards stderr and stdout to logging unless a log handler is set.
bool forward_output_to_log = false;
// If set, these handlers get called automatically whenever stdout/stderr data
// is available. The calls happen on a WORKER THREAD, so the methods have to
// make sure the code is thread-safe. The |data| sent to the output handler
// is NULL terminated.
// If not set, the output is forwarded to the stdout/stderr of the calling
// process.
OutputHandler stdout_handler;
OutputHandler stderr_handler;
// Returns |name| if set, otherwise |command|.
const std::string& Name() const;
};
// Runs a background process and pipes stdin/stdout/stderr.
class Process {
public:
static constexpr uint32_t kExitCodeNotStarted = 4000000000;
static constexpr uint32_t kExitCodeStillRunning = 4000000001;
static constexpr uint32_t kExitCodeFailedToGetExitCode = 4000000002;
explicit Process(const ProcessStartInfo& start_info);
virtual ~Process();
// Start the background process.
virtual absl::Status Start() = 0;
// Runs the process until it exits, an error occurs (see GetStatus()) or
// |exit_condition| returns true.
virtual absl::Status RunUntil(std::function<bool()> exit_condition) = 0;
// Runs the process until it exits or an error occurs (see GetStatus()).
absl::Status RunUntilExit();
// Ends the process.
virtual absl::Status Terminate() = 0;
// Writes |data| of size |size| to the child process stdin. Only applicable
// if |redirect_stdin| was set to true in the start info, no-op if not.
virtual absl::Status WriteToStdIn(const void* data, size_t size) = 0;
// Closes the stdin pipe if |redirect_stdin| was set to true.
virtual void CloseStdIn() = 0;
// Returns true if the process has exited.
virtual bool HasExited() const = 0;
// Returns the process exit code if the process exited and the code could be
// retrieved successfully. Returns |kExitCodeNotStarted| if the process has
// not been started. Returns |kExitCodeStillRunning| if the process has not
// exited yet. Returns |kExitCodeFailedToGetExitCode| if the process exit code
// could not be retrieved.
virtual uint32_t ExitCode() const = 0;
// Returns the internal status of the process. This is an internal status. If
// the process fails with an error, this is not reported in this status, but
// instead in ExitCode().
virtual absl::Status GetStatus() const = 0;
protected:
ProcessStartInfo start_info_;
};
// Abstract process factory.
class ProcessFactory {
public:
virtual ~ProcessFactory();
// Creates a new process with given |start_info|.
virtual std::unique_ptr<Process> Create(
const ProcessStartInfo& start_info) = 0;
// Convenience method that starts a process with the given |start_info| and
// runs it until exit. Returns an error if starting or running the process
// fails, or if the exit code is not 0.
absl::Status Run(const ProcessStartInfo& start_info);
};
// Creates Windows processes.
class WinProcessFactory : public ProcessFactory {
public:
~WinProcessFactory() override;
// ProcessFactory:
std::unique_ptr<Process> Create(const ProcessStartInfo& start_info) override;
};
} // namespace cdc_ft
#endif // COMMON_PROCESS_H_

342
common/process_test.cc Normal file
View File

@@ -0,0 +1,342 @@
// Copyright 2022 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "common/process.h"
// Windows must be included first.
// clang-format off
#define WIN32_LEAN_AND_MEAN
#include <windows.h>
#include <tlhelp32.h>
// clang-format on
#include <atomic>
#include "common/log.h"
#include "common/scoped_handle_win.h"
#include "common/status_test_macros.h"
#include "common/stopwatch.h"
#include "common/util.h"
#include "gtest/gtest.h"
namespace cdc_ft {
namespace {
// Terminates a process by name.
// Returns true if the process was terminated.
// Returns false if the process was not found or if it failed to be terminated.
bool TerminateProcessByName(const char* name) {
PROCESSENTRY32 entry;
entry.dwSize = sizeof(PROCESSENTRY32);
ScopedHandle snapshot(CreateToolhelp32Snapshot(TH32CS_SNAPPROCESS, NULL));
std::wstring wide_name = Util::Utf8ToWideStr(name);
if (Process32First(snapshot.Get(), &entry)) {
do {
if (_wcsicmp(entry.szExeFile, wide_name.c_str()) != 0) continue;
ScopedHandle hProcess(
OpenProcess(PROCESS_ALL_ACCESS, FALSE, entry.th32ProcessID));
if (!TerminateProcess(hProcess.Get(), 0)) {
LOG_ERROR("Failed to terminate process '%s'", name)
return false;
}
return true;
} while (Process32Next(snapshot.Get(), &entry));
}
LOG_ERROR("Failed to find process '%s'", name)
return false;
}
// Filters for "echos" and puts all messages and levels into a vector.
class EchosTestLog : public Log {
public:
explicit EchosTestLog(LogLevel log_level) : Log(log_level) {}
void WriteLogMessage(LogLevel level, const char* file, int line,
const char* func, const char* message) override {
if (strncmp(message, "echos", strlen("echos")) == 0) {
levels.push_back(level);
messages.push_back(message);
}
}
std::vector<LogLevel> levels;
std::vector<std::string> messages;
};
class ProcessTest : public ::testing::Test {
public:
void SetUp() override {
Log::Initialize(std::make_unique<ConsoleLog>(LogLevel::kInfo));
}
void TearDown() override { Log::Shutdown(); }
protected:
WinProcessFactory process_factory_;
};
TEST_F(ProcessTest, ProcessNotStarted) {
ProcessStartInfo start_info;
std::unique_ptr<Process> process = process_factory_.Create(start_info);
EXPECT_NOT_OK(process->RunUntilExit());
EXPECT_EQ(process->ExitCode(), Process::kExitCodeNotStarted);
}
TEST_F(ProcessTest, RunSimpleCommandSucceeds) {
ProcessStartInfo start_info;
start_info.command = "cmd /C \"echo\"";
std::unique_ptr<Process> process = process_factory_.Create(start_info);
EXPECT_OK(process->Start());
EXPECT_OK(process->RunUntilExit());
EXPECT_TRUE(process->HasExited());
EXPECT_EQ(process->ExitCode(), 0u);
}
TEST_F(ProcessTest, RunSimpleCommandFails) {
ProcessStartInfo start_info;
start_info.command = "cmd /C \"dir /INVALID\"";
std::unique_ptr<Process> process = process_factory_.Create(start_info);
EXPECT_OK(process->Start());
EXPECT_OK(process->RunUntilExit());
EXPECT_TRUE(process->HasExited());
EXPECT_EQ(process->ExitCode(), 1u);
}
TEST_F(ProcessTest, RunAndReadStdOut) {
std::string std_out;
std::string message = "hello world";
ProcessStartInfo start_info;
start_info.command = "cmd /C \"echo " + message + "\"";
start_info.stdout_handler = [&std_out](const char* data, size_t) {
std_out += data;
return absl::OkStatus();
};
std::unique_ptr<Process> process = process_factory_.Create(start_info);
EXPECT_OK(process->Start());
EXPECT_OK(process->RunUntilExit());
EXPECT_TRUE(process->HasExited());
EXPECT_EQ(process->ExitCode(), 0u);
EXPECT_EQ(std_out, message + "\r\n");
}
TEST_F(ProcessTest, RunAndReadStdOutAndStdErr) {
std::string std_out;
std::string std_err;
std::string stdout_message = "stdout message";
std::string stderr_message = "stderr message";
ProcessStartInfo start_info;
start_info.command = "cmd /C \"echo " + stdout_message + " & echo " +
stderr_message + " 1>&2\"";
start_info.stdout_handler = [&std_out](const char* data, size_t) {
std_out += data;
return absl::OkStatus();
};
start_info.stderr_handler = [&std_err](const char* data, size_t) {
std_err += data;
return absl::OkStatus();
};
std::unique_ptr<Process> process = process_factory_.Create(start_info);
EXPECT_OK(process->Start());
EXPECT_OK(process->RunUntilExit());
EXPECT_TRUE(process->HasExited());
EXPECT_EQ(process->ExitCode(), 0u);
EXPECT_EQ(std_out, stdout_message + " \r\n");
EXPECT_EQ(std_err, stderr_message + " \r\n");
}
TEST_F(ProcessTest, RunWriteStdInAndReadStdOut) {
std::string std_out;
std::string message = "Foo\nBar\nfoo\nbar\nFar\nBoo";
// Find all lines containing "F".
ProcessStartInfo start_info;
start_info.command = "findstr F";
start_info.redirect_stdin = true;
start_info.stdout_handler = [&std_out](const char* data, size_t) {
std_out += data;
return absl::OkStatus();
};
std::unique_ptr<Process> process = process_factory_.Create(start_info);
EXPECT_OK(process->Start());
EXPECT_OK(process->WriteToStdIn(message.data(), message.size()));
process->CloseStdIn();
EXPECT_OK(process->RunUntilExit());
EXPECT_TRUE(process->HasExited());
EXPECT_EQ(process->ExitCode(), 0u);
EXPECT_EQ(std_out, "Foo\nFar\n");
}
TEST_F(ProcessTest, RunIoStressTest) {
std::string std_out;
std::string expected_stdout;
// Find all lines containing "1".
ProcessStartInfo start_info;
start_info.command = "findstr 1";
start_info.redirect_stdin = true;
start_info.stdout_handler = [&std_out](const char* data, size_t) {
std_out += data;
return absl::OkStatus();
};
std::unique_ptr<Process> process = process_factory_.Create(start_info);
EXPECT_OK(process->Start());
// Write lots of lines to stdin.
// Every other line starts with '1' and should be picked up by findstr.
std::string message(2048, 'x');
message.back() = '\n';
const int num_lines = 1000;
for (int n = 0; n < num_lines; ++n) {
message[0] = n & 1 ? '1' : '2';
if (n & 1) {
expected_stdout += message;
}
EXPECT_OK(process->WriteToStdIn(message.data(), message.size()));
}
process->CloseStdIn();
EXPECT_OK(process->RunUntilExit());
EXPECT_TRUE(process->HasExited());
EXPECT_EQ(process->ExitCode(), 0u);
EXPECT_EQ(std_out, expected_stdout);
}
TEST_F(ProcessTest, RunUntil) {
std::string std_out;
// Find all lines containing "msg". "findstr" is a convenient command that
// echoes stdin to stdout for matching lines. "echo" doesn't work with stdin.
ProcessStartInfo start_info;
start_info.command = "findstr stop";
start_info.redirect_stdin = true;
std::atomic_bool stop(false);
start_info.stdout_handler = [&std_out, &stop](const char* data, size_t) {
// Check whether someone sent the "stop" command.
// Note: This runs in a background thread.
std_out += data;
stop = std_out.find("stop") != std::string::npos;
return absl::OkStatus();
};
std::unique_ptr<Process> process = process_factory_.Create(start_info);
EXPECT_OK(process->Start());
// Send "msg stop" and check that it stops.
std::string message = " stop\n";
EXPECT_OK(process->WriteToStdIn(message.data(), message.size()));
process->CloseStdIn();
EXPECT_OK(process->RunUntil([&stop]() { return stop.load(); }));
EXPECT_TRUE(stop);
EXPECT_OK(process->RunUntilExit());
EXPECT_TRUE(process->HasExited());
EXPECT_EQ(process->ExitCode(), 0u);
}
TEST_F(ProcessTest, ForwardOutputToLogging) {
Log::Shutdown();
auto log_ptr = std::make_unique<EchosTestLog>(LogLevel::kInfo);
EchosTestLog* log = log_ptr.get();
Log::Initialize(std::move(log_ptr));
const std::string stdout_message = "stdout message";
const std::string stderr_message = "stderr message";
ProcessStartInfo start_info;
start_info.command = "cmd /C \"echo " + stdout_message + " & echo " +
stderr_message + " 1>&2\"";
start_info.forward_output_to_log = true;
start_info.name = "echos";
EXPECT_OK(process_factory_.Run(start_info));
ASSERT_EQ(log->messages.size(), 2);
EXPECT_EQ(log->messages[0], "echos_stdout: " + stdout_message + " ");
EXPECT_EQ(log->messages[1], "echos_stderr: " + stderr_message + " ");
}
TEST_F(ProcessTest, LogOutputLevelDetection) {
Log::Shutdown();
auto log_ptr = std::make_unique<EchosTestLog>(LogLevel::kVerbose);
EchosTestLog* log = log_ptr.get();
Log::Initialize(std::move(log_ptr));
ProcessStartInfo start_info;
start_info.command = "cmd /C \"echo VERBOSE msg1 && ";
start_info.command += "echo DEBUG msg2 && ";
start_info.command += "echo INFO msg3 && ";
start_info.command += "echo WARNING msg4 && ";
start_info.command += "echo ERROR msg5 && ";
start_info.command += "echo msg6\"";
start_info.forward_output_to_log = true;
start_info.name = "echos";
EXPECT_OK(process_factory_.Run(start_info));
ASSERT_EQ(log->messages.size(), 6);
ASSERT_EQ(log->levels.size(), 6);
EXPECT_EQ(log->messages[0], "echos_stdout: VERBOSE msg1 ");
EXPECT_EQ(log->messages[1], "echos_stdout: DEBUG msg2 ");
EXPECT_EQ(log->messages[2], "echos_stdout: INFO msg3 ");
EXPECT_EQ(log->messages[3], "echos_stdout: WARNING msg4 ");
EXPECT_EQ(log->messages[4], "echos_stdout: ERROR msg5 ");
EXPECT_EQ(log->messages[5], "echos_stdout: msg6");
EXPECT_EQ(log->levels[0], LogLevel::kVerbose);
EXPECT_EQ(log->levels[1], LogLevel::kDebug);
EXPECT_EQ(log->levels[2], LogLevel::kInfo);
EXPECT_EQ(log->levels[3], LogLevel::kWarning);
EXPECT_EQ(log->levels[4], LogLevel::kError);
EXPECT_EQ(log->levels[5], LogLevel::kInfo);
}
TEST_F(ProcessTest, Terminate) {
ProcessStartInfo start_info;
start_info.command = "timeout /T 30";
std::unique_ptr<Process> process = process_factory_.Create(start_info);
EXPECT_OK(process->Start());
EXPECT_EQ(process->ExitCode(), Process::kExitCodeStillRunning);
EXPECT_OK(process->Terminate());
EXPECT_EQ(process->ExitCode(), Process::kExitCodeNotStarted);
}
TEST_F(ProcessTest, TerminateAlreadyExited) {
ProcessStartInfo start_info;
start_info.command = "timeout /T 30";
std::unique_ptr<Process> process = process_factory_.Create(start_info);
EXPECT_OK(process->Start());
EXPECT_FALSE(process->HasExited());
bool terminated = false;
Stopwatch sw;
while (sw.ElapsedSeconds() < 5 && !terminated) {
terminated = TerminateProcessByName("timeout.exe");
if (!terminated) Util::Sleep(1);
}
EXPECT_TRUE(terminated);
EXPECT_OK(process->Terminate());
}
} // namespace
} // namespace cdc_ft

848
common/process_win.cc Normal file
View File

@@ -0,0 +1,848 @@
// Copyright 2022 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "common/process.h"
#define WIN32_LEAN_AND_MEAN
#include <windows.h>
#include <atomic>
#include <cassert>
#include <mutex>
#include <thread>
#include <vector>
#include "absl/strings/str_format.h"
#include "common/scoped_handle_win.h"
#include "common/status.h"
#include "common/util.h"
namespace cdc_ft {
namespace {
// SetThreadDescription is not available on all Windows versions,
// e.g. Win Server 2016, which happens to run on YRunners!.
typedef HRESULT(WINAPI* SetThreadDescription)(HANDLE hThread,
PCWSTR lpThreadDescription);
// Sets the name of the current thread to |name|.
// Works from Windows 10 version 1607 on, no-op otherwise.
void SetThreadName(const std::string& name) {
static auto set_thread_description_func =
reinterpret_cast<SetThreadDescription>(::GetProcAddress(
::GetModuleHandle(L"Kernel32.dll"), "SetThreadDescription"));
if (set_thread_description_func) {
set_thread_description_func(::GetCurrentThread(),
Util::Utf8ToWideStr(name).c_str());
}
}
std::atomic_int g_pipe_serial_number(0);
// Creates a pipe suitable for overlapped IO. Regular anonymous pipes in Windows
// don't support overlapped IO. This method creates a named pipe with a unique
// name, sets it up for overlapped IO and returns read/write ends.
absl::Status CreatePipeForOverlappedIo(ScopedHandle* pipe_read_end,
ScopedHandle* pipe_write_end) {
// We need named pipes for overlapped IO, so create a unique name.
int id = g_pipe_serial_number++;
std::string pipe_name = absl::StrFormat(
R"(\\.\Pipe\GgpRsyncIoPipe.%08x.%08x)", GetCurrentProcessId(), id);
// Set the bInheritHandle flag so pipe handles are inherited.
SECURITY_ATTRIBUTES security_attributes;
security_attributes.nLength = sizeof(SECURITY_ATTRIBUTES);
security_attributes.bInheritHandle = TRUE;
security_attributes.lpSecurityDescriptor = nullptr;
*pipe_read_end = ScopedHandle(CreateNamedPipeA(
pipe_name.c_str(), PIPE_ACCESS_INBOUND | FILE_FLAG_OVERLAPPED,
PIPE_TYPE_BYTE | PIPE_WAIT,
1, // Number of pipes
4096, // Out buffer size
4096, // In buffer size
120 * 1000, // Timeout in ms
&security_attributes));
if (!pipe_read_end->IsValid()) {
return MakeStatus("Failed to create pipe read end: %s",
Util::GetLastWin32Error());
}
*pipe_write_end =
ScopedHandle(CreateFileA(pipe_name.c_str(), GENERIC_WRITE,
0, // No sharing
&security_attributes, OPEN_EXISTING,
FILE_ATTRIBUTE_NORMAL | FILE_FLAG_OVERLAPPED,
nullptr)); // Template file
if (!pipe_write_end->IsValid()) {
// Note that Close() might change GetLastErrorString()!
absl::Status status = MakeStatus("Failed to create pipe write end: %s",
Util::GetLastWin32Error());
pipe_read_end->Close();
return status;
}
return absl::OkStatus();
}
// Creates a pipe intended for piping stdin from this process to a child
// process. The read end is inherited to the child process (so it can read
// stdin from it), the write end is not. The pipe is NOT suitable for async IO.
absl::Status SetUpInputPipe(ScopedHandle* read_end, ScopedHandle* write_end) {
// Set the bInheritHandle flag so pipe handles are inherited.
SECURITY_ATTRIBUTES security_attributes;
security_attributes.nLength = sizeof(SECURITY_ATTRIBUTES);
security_attributes.bInheritHandle = TRUE;
security_attributes.lpSecurityDescriptor = nullptr;
// Create pipe.
HANDLE read_end_handle, write_end_handle;
if (!CreatePipe(&read_end_handle, &write_end_handle, &security_attributes,
4096)) {
return MakeStatus("Failed to create input pipes");
}
*read_end = ScopedHandle(read_end_handle);
*write_end = ScopedHandle(write_end_handle);
// Ensure the write end of the pipe is not inherited to the child process.
if (!SetHandleInformation(write_end->Get(), HANDLE_FLAG_INHERIT, 0)) {
// Note that Close() might change GetLastErrorString()!
absl::Status status = MakeStatus("Failed to set handle information: %s",
Util::GetLastWin32Error());
read_end->Close();
write_end->Close();
return status;
}
return absl::OkStatus();
}
// Creates a pipe intended for piping stdout/stderr from a child process to this
// process. The write end is inherited to the child process (so it can write
// stdout/stderr to it), the read end is not. The pipe is suitable for async IO.
absl::Status SetUpOutputPipe(ScopedHandle* read_end, ScopedHandle* write_end) {
// Create pipe.
absl::Status status = CreatePipeForOverlappedIo(read_end, write_end);
if (!status.ok()) {
return WrapStatus(status, "Failed to create output pipes");
}
// Ensure the read end of the pipe is not inherited to the child process.
if (!SetHandleInformation(read_end->Get(), HANDLE_FLAG_INHERIT, 0)) {
// Note that Close() might change GetLastErrorString()!
status = MakeStatus("Failed to set handle information: %s",
Util::GetLastWin32Error());
read_end->Close();
write_end->Close();
return status;
}
return absl::OkStatus();
}
// Helper class for performing async IO from the stdout/stderr pipes created by
// SetUpOutputPipe().
class AsyncReader {
public:
using OutputHandler = ProcessStartInfo::OutputHandler;
AsyncReader(HANDLE pipe_handle, OutputHandler output_handler)
: pipe_handle_(pipe_handle),
output_handler_(std::move(output_handler)),
buffer_(4096) {
ZeroMemory(&overlapped_, sizeof(overlapped_));
}
~AsyncReader() {
// Better cancel pending IO before the buffers get deleted. I heard from a
// "friend" that they got a heap corruption when they didn't do it.
absl::Status status = CancelPendingIo();
if (!status.ok()) {
LOG_WARNING("%s", status.ToString());
}
}
// Returns the event that is triggered when async IO completes.
HANDLE GetEvent() const { return event_.Get(); }
// Initialize the IO event (see GetEvent()) and issue an async IO request.
absl::Status Initialize() {
// Create signaled manual reset event.
event_ = ScopedHandle(CreateEvent(nullptr, TRUE, TRUE, nullptr));
if (!event_.IsValid()) {
return MakeStatus("CreateEvent failed: %s", Util::GetLastWin32Error());
}
overlapped_.hEvent = event_.Get();
// Start reading.
absl::Status status = IssueRead();
if (!status.ok()) {
return WrapStatus(status, "IssueRead() failed");
}
return absl::OkStatus();
}
// Reads the result of the async IO request. Async IO must be pending.
// Should be called if the IO event (see GetEvent()) was triggered, otherwise
// the method will block until the async IO result is available.
absl::Status Read() {
assert(io_pending_);
io_pending_ = false;
DWORD num_bytes_read;
if (!GetOverlappedResult(pipe_handle_, &overlapped_, &num_bytes_read,
true)) {
switch (GetLastError()) {
case ERROR_BROKEN_PIPE:
// The pipe was closed by the child process. Set the EOF() marker.
LOG_VERBOSE("EOF");
eof_ = true;
break;
default:
return MakeStatus("GetOverlappedResult() failed: %s",
Util::GetLastWin32Error());
}
} else {
// Async IO succeeded. Append null terminator in case the handler accesses
// the data like a C string.
assert(num_bytes_read < buffer_.size());
buffer_[num_bytes_read] = 0;
absl::Status status = output_handler_(buffer_.data(), num_bytes_read);
if (!status.ok()) {
// Don't return an error, it stops the pump thread and leads to freezes.
LOG_DEBUG("%s", WrapStatus(status, "Output handler failed").ToString());
}
}
if (!ResetEvent(event_.Get())) {
return MakeStatus("ResetEvent() failed: %s", Util::GetLastWin32Error());
}
// Only issue a new read if the pipe is still open.
if (!eof_) {
absl::Status status = IssueRead();
if (!status.ok()) {
return WrapStatus(status, "IssueRead() failed");
}
}
return absl::OkStatus();
}
// Cancels the currently pending async IO request if there is any.
// Must be called from the same thread as Initialize() and Read().
absl::Status CancelPendingIo() {
if (!io_pending_) {
return absl::OkStatus();
}
if (!CancelIo(pipe_handle_)) {
return MakeStatus(
"CancelIo() failed. If you get a heap corruption, this is why.");
}
io_pending_ = false;
return absl::OkStatus();
}
private:
// Queues a new async IO request. If data is already available, immediately
// calls the output handler. Returns false on error (failing output handler,
// IO error).
absl::Status IssueRead() {
assert(!io_pending_);
// The pipe might already contain data that can be read synchronously. Just
// keep reading it.
DWORD num_bytes_read;
while (ReadFile(pipe_handle_, buffer_.data(),
static_cast<DWORD>(buffer_.size()) - 1, &num_bytes_read,
&overlapped_)) {
// Append a null terminator in case handler interprets the data as C str.
assert(num_bytes_read < buffer_.size());
buffer_[num_bytes_read] = 0;
absl::Status status = output_handler_(buffer_.data(), num_bytes_read);
if (!status.ok()) {
// Don't return an error, it stops the pump thread and leads to freezes.
LOG_DEBUG("%s", WrapStatus(status, "Output handler failed").ToString());
}
}
switch (GetLastError()) {
case ERROR_IO_PENDING:
// Async IO in progress, this is expected. The caller should wait for
// the event (see GetEvent()) to retrieve the data.
io_pending_ = true;
return absl::OkStatus();
case ERROR_BROKEN_PIPE:
// The pipe was closed by the child process. Set the EOF() marker.
LOG_VERBOSE("EOF");
eof_ = true;
return absl::OkStatus();
default:
return MakeStatus("ReadFile failed: %s", Util::GetLastWin32Error());
}
}
// Not owned.
HANDLE pipe_handle_;
OutputHandler output_handler_;
std::vector<char> buffer_;
OVERLAPPED overlapped_;
ScopedHandle event_;
bool eof_ = false;
bool io_pending_ = false;
};
struct ProcessInfo {
PROCESS_INFORMATION pi;
ScopedHandle job;
ScopedHandle stdin_write_end;
ScopedHandle stdout_read_end;
ScopedHandle stderr_read_end;
ProcessInfo() { ZeroMemory(&pi, sizeof(pi)); }
};
// Background thread to read stdout/stderr from the child process.
// Also watches the child process for exit.
class MessagePumpThread {
public:
MessagePumpThread(const ProcessInfo& process_info,
const ProcessStartInfo& start_info)
: process_handle_(process_info.pi.hProcess), name_(start_info.Name()) {
// Initialize stdout reader if necessary.
if (process_info.stdout_read_end.IsValid()) {
stdout_reader_ = std::make_unique<AsyncReader>(
process_info.stdout_read_end.Get(), start_info.stdout_handler);
}
// Initialize stderr reader if necessary.
if (process_info.stderr_read_end.IsValid()) {
stderr_reader_ = std::make_unique<AsyncReader>(
process_info.stderr_read_end.Get(), start_info.stderr_handler);
}
// Create manual reset event that is not signaled.
shutdown_event_ = ScopedHandle(CreateEvent(nullptr, TRUE, FALSE, nullptr));
worker_ = std::thread([this]() { ThreadWorkerMain(); });
}
~MessagePumpThread() { Shutdown(); }
void Shutdown() {
if (shutdown_event_.IsValid() && !SetEvent(shutdown_event_.Get())) {
// Can't do much if this fails.
LOG_ERROR("Shutting down process message thread failed");
exit(1);
}
if (worker_.joinable()) {
worker_.join();
}
}
// Contains the error message if some error occurred and the message pump
// thread was shut down.
absl::Status GetStatus() {
std::lock_guard<std::mutex> lock(status_mutex_);
return status_;
}
// Returns true if the process has exited.
bool HasExited() {
std::lock_guard<std::mutex> lock(exit_mutex_);
return has_exited_;
}
// Returns the process exit code.
uint32_t ExitCode() {
std::lock_guard<std::mutex> lock(exit_mutex_);
return exit_code_;
}
private:
void SetStatus(absl::Status status) {
LOG_DEBUG("Setting status %s of process %s", status.ToString().c_str(),
name_.c_str());
std::lock_guard<std::mutex> lock(status_mutex_);
status_ = status;
}
void ThreadWorkerMain() {
SetThreadName(name_);
LOG_VERBOSE("Process message thread started: %s", name_.c_str());
if (!shutdown_event_.IsValid()) {
SetStatus(
MakeStatus("CreateEvent failed: %s", Util::GetLastWin32Error()));
return;
}
// Be sure to call AsyncReader::Initialize() from this thread, so all
// AsyncIO stays here.
// Initialize stdout reader if present (schedules AsyncIO).
if (stdout_reader_) {
absl::Status init_status = stdout_reader_->Initialize();
if (!init_status.ok()) {
SetStatus(
WrapStatus(init_status, "Failed to initialize stdout reader"));
return;
}
}
// Initialize stderr reader if present (schedules AsyncIO).
if (stderr_reader_) {
absl::Status init_status = stderr_reader_->Initialize();
if (!init_status.ok()) {
SetStatus(
WrapStatus(init_status, "Failed to initialize stderr reader"));
return;
}
}
// Initialize handles to watch.
std::vector<HANDLE> watch_handles;
size_t process_index = watch_handles.size();
watch_handles.push_back(process_handle_);
size_t shutdown_index = watch_handles.size();
watch_handles.push_back(shutdown_event_.Get());
size_t stdout_index = SIZE_MAX;
if (stdout_reader_) {
stdout_index = watch_handles.size();
watch_handles.push_back(stdout_reader_->GetEvent());
}
size_t stderr_index = SIZE_MAX;
if (stderr_reader_) {
stderr_index = watch_handles.size();
watch_handles.push_back(stderr_reader_->GetEvent());
}
for (;;) {
const uint32_t wait_index =
WaitForMultipleObjects(static_cast<DWORD>(watch_handles.size()),
watch_handles.data(), false, UINT_MAX);
if (wait_index == WAIT_TIMEOUT) {
continue;
}
const uint32_t handle_index = wait_index - WAIT_OBJECT_0;
if (handle_index >= watch_handles.size()) {
SetStatus(MakeStatus(
"WaitForMultipleObjects failed with invalid handle index %u",
handle_index));
return;
}
if (handle_index == process_index) {
// Process exited.
std::lock_guard<std::mutex> lock(exit_mutex_);
has_exited_ = true;
// Get process exit code.
if (!GetExitCodeProcess(process_handle_, &exit_code_)) {
LOG_WARNING("Failed to get exit code for process '%s': %s", name_,
Util::GetLastWin32Error());
exit_code_ = Process::kExitCodeFailedToGetExitCode;
}
break;
}
if (handle_index == shutdown_index) {
// Shutdown() was called.
break;
}
if (handle_index == stdout_index) {
// Data on stdout is available.
absl::Status status = stdout_reader_->Read();
if (!status.ok()) {
SetStatus(WrapStatus(status, "Failed to read stdout"));
return;
}
continue;
}
if (handle_index == stderr_index) {
// Data on stderr is available.
absl::Status status = stderr_reader_->Read();
if (!status.ok()) {
SetStatus(WrapStatus(status, "Failed to read stderr"));
return;
}
continue;
}
}
// Cancel any pending IO.
stdout_reader_.reset();
stderr_reader_.reset();
LOG_VERBOSE("Process message thread stopped: %s", name_.c_str());
}
HANDLE process_handle_;
std::string name_;
std::thread worker_;
ScopedHandle shutdown_event_;
absl::Status status_;
std::mutex status_mutex_;
bool has_exited_ = false;
DWORD exit_code_ = Process::kExitCodeStillRunning;
std::mutex exit_mutex_;
std::unique_ptr<AsyncReader> stdout_reader_;
std::unique_ptr<AsyncReader> stderr_reader_;
};
// Try to guess the log level from |str|, e.g. LogLevel::kError if |str| starts
// with "ERROR".
LogLevel GuessLogLevel(const char* str) {
if (strncmp(str, "ERROR", 5) == 0)
return LogLevel::kError;
else if (strncmp(str, "WARNING", 7) == 0)
return LogLevel::kWarning;
else if (strncmp(str, "DEBUG", 5) == 0)
return LogLevel::kDebug;
else if (strncmp(str, "VERBOSE", 7) == 0)
return LogLevel::kVerbose;
return LogLevel::kInfo;
}
} // namespace
absl::Status LogOutput(const char* name, const char* data, size_t /*size*/,
absl::optional<LogLevel> log_level) {
const char* newline_pos = strpbrk(data, "\r\n");
while (newline_pos) {
if (newline_pos > data) {
LOG_LEVEL(log_level ? *log_level : GuessLogLevel(data), "%s: %s", name,
std::string(data, newline_pos - data));
}
data = newline_pos + 1;
newline_pos = strpbrk(data, "\r\n");
}
// There's always guaranteed to be NULL terminator, even if |size| is 0.
// Note that the rest here might be an incomplete line, but we print it,
// anyway, to not risk loosing data.
if (data[0] != 0) {
LOG_LEVEL(log_level ? *log_level : GuessLogLevel(data), "%s: %s", name,
data);
}
return absl::OkStatus();
}
const std::string& ProcessStartInfo::Name() const {
return !name.empty() ? name : command;
}
Process::Process(const ProcessStartInfo& start_info)
: start_info_(start_info) {}
Process::~Process() = default;
absl::Status Process::RunUntilExit() {
return RunUntil([]() { return false; });
}
// Implementation of a Windows process.
class WinProcess : public Process {
public:
explicit WinProcess(const ProcessStartInfo& start_info);
~WinProcess() override;
// Process:
absl::Status Start() override;
absl::Status RunUntil(std::function<bool()> exit_condition) override;
absl::Status Terminate() override;
absl::Status WriteToStdIn(const void* data, size_t size) override;
void CloseStdIn() override;
bool HasExited() const override;
uint32_t ExitCode() const override;
absl::Status GetStatus() const override;
private:
std::unique_ptr<ProcessInfo> process_info_;
std::unique_ptr<MessagePumpThread> message_pump_;
};
WinProcess::WinProcess(const ProcessStartInfo& start_info)
: Process(start_info) {}
WinProcess::~WinProcess() { Terminate().IgnoreError(); }
absl::Status WinProcess::Start() {
LOG_INFO("Starting process %s", start_info_.command.c_str());
std::wstring command = Util::Utf8ToWideStr(start_info_.command);
wchar_t* command_cstr = const_cast<wchar_t*>(command.c_str());
STARTUPINFO si;
ZeroMemory(&si, sizeof(si));
si.cb = sizeof(si);
process_info_ = std::make_unique<ProcessInfo>();
// Create stdout pipes if necessary.
ScopedHandle stdin_read_end;
if (start_info_.redirect_stdin) {
absl::Status status =
SetUpInputPipe(&stdin_read_end, &process_info_->stdin_write_end);
if (!status.ok()) {
return WrapStatus(status, "Failed to set up stdin pipes");
}
}
// Set up handlers to forward output to logging.
if (!start_info_.stdout_handler && start_info_.forward_output_to_log) {
start_info_.stdout_handler = [name = start_info_.Name() + "_stdout"](
const char* data, size_t size) {
return LogOutput(name.c_str(), data, size);
};
}
if (!start_info_.stderr_handler && start_info_.forward_output_to_log) {
start_info_.stderr_handler = [name = start_info_.Name() + "_stderr"](
const char* data, size_t size) {
return LogOutput(name.c_str(), data, size, LogLevel::kError);
};
}
// Create stdout pipes if necessary.
ScopedHandle stdout_write_end;
if (start_info_.stdout_handler) {
absl::Status status =
SetUpOutputPipe(&process_info_->stdout_read_end, &stdout_write_end);
if (!status.ok()) {
return WrapStatus(status, "Failed to set up stdout pipes");
}
}
// Create stderr pipes if necessary.
ScopedHandle stderr_write_end;
if (start_info_.stderr_handler) {
absl::Status status =
SetUpOutputPipe(&process_info_->stderr_read_end, &stderr_write_end);
if (!status.ok()) {
return WrapStatus(status, "Failed to set up stderr pipes");
}
}
// Set up handle redirection. Note that it's not possible to redirect only
// some handles, so use GetStdHandle() for the ones not redirected.
si.dwFlags |= STARTF_USESTDHANDLES;
si.hStdInput = stdin_read_end.IsValid() ? stdin_read_end.Get()
: GetStdHandle(STD_INPUT_HANDLE);
si.hStdOutput = stdout_write_end.IsValid() ? stdout_write_end.Get()
: GetStdHandle(STD_OUTPUT_HANDLE);
si.hStdError = stderr_write_end.IsValid() ? stderr_write_end.Get()
: GetStdHandle(STD_ERROR_HANDLE);
// Make sure the process gets closed if the parent (this!) process exits.
process_info_->job = ScopedHandle(CreateJobObject(NULL, NULL));
if (!process_info_->job.IsValid()) {
return MakeStatus("CreateJobObject() failed: %s",
Util::GetLastWin32Error());
}
JOBOBJECT_EXTENDED_LIMIT_INFORMATION jeli = {0};
jeli.BasicLimitInformation.LimitFlags = JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE;
bool success = SetInformationJobObject(process_info_->job.Get(),
JobObjectExtendedLimitInformation,
&jeli, sizeof(jeli));
if (!success) {
return MakeStatus("SetInformationJobObject() failed: %s",
Util::GetLastWin32Error());
}
// Start the child process.
success = CreateProcess(NULL, // No module name (use command line)
command_cstr,
NULL, // Process handle not inheritable
NULL, // Thread handle not inheritable
TRUE, // Inherit handles
0, // No creation flags
NULL, // Use parent's environment block
NULL, // Use parent's starting directory
&si, &process_info_->pi);
if (!success) {
return MakeStatus("CreateProcess() failed: %s", Util::GetLastWin32Error());
}
success = AssignProcessToJobObject(process_info_->job.Get(),
process_info_->pi.hProcess);
if (!success) {
return MakeStatus("AssignProcessToJobObject() failed: %s",
Util::GetLastWin32Error());
}
// Explicitly close our copies of the input read end and output write ends.
// The child process still has a copy.
stdin_read_end.Close();
stdout_write_end.Close();
stderr_write_end.Close();
// Start message pump thread.
message_pump_ =
std::make_unique<MessagePumpThread>(*process_info_, start_info_);
return absl::OkStatus();
}
absl::Status WinProcess::RunUntil(std::function<bool()> exit_condition) {
if (!process_info_) {
return MakeStatus("Process was never started");
}
// Wait until exit condition is fulfilled.
while (!exit_condition()) {
// Poll |message_pump_|. This is not super performance critical, so a simple
// sleep should do.
Util::Sleep(10);
absl::Status status = message_pump_->GetStatus();
if (!status.ok()) return status;
if (HasExited()) return absl::OkStatus();
}
return absl::OkStatus();
}
absl::Status WinProcess::WriteToStdIn(const void* data, size_t size) {
if (!start_info_.redirect_stdin) {
return absl::FailedPreconditionError("Stdin not redirected");
}
DWORD bytes_written = 0;
assert(size <= UINT32_MAX);
const DWORD dw_size = static_cast<DWORD>(size);
if (!WriteFile(process_info_->stdin_write_end.Get(), data, dw_size,
&bytes_written, nullptr)) {
return MakeStatus("WriteFile() failed: %s", Util::GetLastWin32Error());
}
if (bytes_written != size) {
return MakeStatus("WriteFile() failed: Only %u / %u bytes written",
bytes_written, size);
}
return absl::OkStatus();
}
void WinProcess::CloseStdIn() {
assert(start_info_.redirect_stdin);
process_info_->stdin_write_end.Close();
}
bool WinProcess::HasExited() const {
return message_pump_ ? message_pump_->HasExited() : false;
}
uint32_t WinProcess::ExitCode() const {
return message_pump_ ? message_pump_->ExitCode() : kExitCodeNotStarted;
}
absl::Status WinProcess::GetStatus() const {
return message_pump_ ? message_pump_->GetStatus() : absl::OkStatus();
}
absl::Status WinProcess::Terminate() {
// Stop message pump.
bool should_terminate = false;
if (message_pump_) {
should_terminate = !message_pump_->HasExited();
message_pump_.reset();
}
if (process_info_) {
bool result = true;
if (should_terminate) {
result = TerminateProcess(process_info_->pi.hProcess, 0);
if (!result && GetLastError() == ERROR_ACCESS_DENIED) {
// This means that the process has already exited, but in a way that
// the exit wasn't properly reported to this code (e.g. the process got
// killed somewhere). Just handle this silently.
LOG_DEBUG("Process '%s' already exited", start_info_.Name());
result = true;
}
}
// Close the handles that are not scoped handles.
ScopedHandle(process_info_->pi.hProcess).Close();
ScopedHandle(process_info_->pi.hThread).Close();
process_info_.reset();
if (!result) {
return MakeStatus("TerminateProcess() failed: %s",
Util::GetLastWin32Error());
}
}
return absl::OkStatus();
}
ProcessFactory::~ProcessFactory() = default;
absl::Status ProcessFactory::Run(const ProcessStartInfo& start_info) {
std::unique_ptr<Process> process = Create(start_info);
absl::Status status = process->Start();
if (!status.ok()) {
return WrapStatus(status, "Failed to start process '%s'",
start_info.Name());
}
status = process->RunUntilExit();
if (!status.ok()) {
return WrapStatus(status, "Failed to run process '%s'", start_info.Name());
}
uint32_t exit_code = process->ExitCode();
if (exit_code != 0) {
return MakeStatus("Process '%s' exited with code %u", start_info.Name(),
exit_code);
}
return absl::OkStatus();
}
WinProcessFactory::~WinProcessFactory() = default;
std::unique_ptr<Process> WinProcessFactory::Create(
const ProcessStartInfo& start_info) {
return std::unique_ptr<Process>(new WinProcess(start_info));
}
} // namespace cdc_ft

229
common/remote_util.cc Normal file
View File

@@ -0,0 +1,229 @@
// Copyright 2022 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "common/remote_util.h"
#include <atomic>
#include <regex>
#include <sstream>
#include "absl/strings/str_format.h"
#include "common/path.h"
#include "common/status.h"
namespace cdc_ft {
namespace {
// Escapes command line argument for the Microsoft command line parser in
// preparation for quoting. Double quotes are backslash-escaped. Literal
// backslashes are backslash-escaped if they are followed by a double quote, or
// if they are part of a sequence of backslashes that are followed by a double
// quote.
std::string EscapeForWindows(const std::string& argument) {
std::string str =
std::regex_replace(argument, std::regex(R"(\\*(?=""|$))"), "$1$1");
return std::regex_replace(str, std::regex("\""), "\\\"");
}
// Quotes and escapes a command line argument following the convention
// understood by the Microsoft command line parser.
std::string QuoteArgument(const std::string& argument) {
return absl::StrFormat("\"%s\"", EscapeForWindows(argument));
}
// Quotes and escapes a command line arguments for use in ssh command. The
// argument is first escaped and quoted for Linux using single quotes and then
// it is escaped to be used by the Microsoft command line parser.
std::string QuoteAndEscapeArgumentForSsh(const std::string& argument) {
std::string quoted_argument = absl::StrFormat(
"'%s'", std::regex_replace(argument, std::regex("'"), "'\\''"));
return EscapeForWindows(quoted_argument);
}
// Gets the argument for SSH (reverse) port forwarding, e.g. -L23:localhost:45.
std::string GetPortForwardingArg(int local_port, int remote_port,
bool reverse) {
if (reverse)
return absl::StrFormat("-R%i:localhost:%i ", remote_port, local_port);
return absl::StrFormat("-L%i:localhost:%i ", local_port, remote_port);
}
} // namespace
RemoteUtil::RemoteUtil(int verbosity, bool quiet,
ProcessFactory* process_factory,
bool forward_output_to_log)
: verbosity_(verbosity),
quiet_(quiet),
process_factory_(process_factory),
forward_output_to_log_(forward_output_to_log) {}
void RemoteUtil::SetIpAndPort(const std::string& gamelet_ip, int ssh_port) {
gamelet_ip_ = gamelet_ip;
ssh_port_ = ssh_port;
}
absl::Status RemoteUtil::Scp(std::vector<std::string> source_filepaths,
const std::string& dest, bool compress) {
absl::Status status = CheckIpPort();
if (!status.ok()) {
return status;
}
std::string source_args;
for (const std::string& sourceFilePath : source_filepaths) {
source_args += QuoteArgument(sourceFilePath) + " ";
}
// -p preserves timestamps. This enables timestamp-based up-to-date checks.
ProcessStartInfo start_info;
start_info.command = absl::StrFormat(
"%s "
"%s %s -p -T "
"-F %s "
"-i %s -P %i "
"-oStrictHostKeyChecking=yes "
"-oUserKnownHostsFile=\"\"\"%s\"\"\" %s "
"cloudcast@%s:"
"%s",
QuoteArgument(sdk_util_.GetScpExePath()),
quiet_ || verbosity_ < 2 ? "-q" : "", compress ? "-C" : "",
QuoteArgument(sdk_util_.GetSshConfigPath()),
QuoteArgument(sdk_util_.GetSshKeyFilePath()), ssh_port_,
sdk_util_.GetSshKnownHostsFilePath(), source_args,
QuoteArgument(gamelet_ip_), QuoteAndEscapeArgumentForSsh(dest));
start_info.name = "scp";
start_info.forward_output_to_log = forward_output_to_log_;
return process_factory_->Run(start_info);
}
absl::Status RemoteUtil::Sync(std::vector<std::string> source_filepaths,
const std::string& dest) {
absl::Status status = CheckIpPort();
if (!status.ok()) {
return status;
}
std::string source_args;
for (const std::string& sourceFilePath : source_filepaths) {
source_args += QuoteArgument(sourceFilePath) + " ";
}
ProcessStartInfo start_info;
start_info.command = absl::StrFormat(
"%s --ip=%s --port=%i -z %s %s%s",
path::Join(sdk_util_.GetDevBinPath(), "cdc_rsync"),
QuoteArgument(gamelet_ip_), ssh_port_,
quiet_ || verbosity_ < 2 ? "-q " : " ", source_args, QuoteArgument(dest));
start_info.name = "cdc_rsync";
start_info.forward_output_to_log = forward_output_to_log_;
return process_factory_->Run(start_info);
}
absl::Status RemoteUtil::Chmod(const std::string& mode,
const std::string& remote_path, bool quiet) {
std::string remote_command = absl::StrFormat(
"chmod %s %s %s", QuoteArgument(mode),
QuoteAndEscapeArgumentForSsh(remote_path), quiet ? "-f" : "");
return Run(remote_command, "chmod");
}
absl::Status RemoteUtil::Rm(const std::string& remote_path, bool force) {
std::string remote_command = absl::StrFormat(
"rm %s %s", force ? "-f" : "", QuoteAndEscapeArgumentForSsh(remote_path));
return Run(remote_command, "rm");
}
absl::Status RemoteUtil::Mv(const std::string& old_remote_path,
const std::string& new_remote_path) {
std::string remote_command =
absl::StrFormat("mv %s %s", QuoteAndEscapeArgumentForSsh(old_remote_path),
QuoteAndEscapeArgumentForSsh(new_remote_path));
return Run(remote_command, "mv");
}
absl::Status RemoteUtil::Run(std::string remote_command, std::string name) {
absl::Status status = CheckIpPort();
if (!status.ok()) {
return status;
}
ProcessStartInfo start_info =
BuildProcessStartInfoForSsh(std::move(remote_command));
start_info.name = std::move(name);
start_info.forward_output_to_log = forward_output_to_log_;
return process_factory_->Run(start_info);
}
ProcessStartInfo RemoteUtil::BuildProcessStartInfoForSsh(
std::string remote_command) {
return BuildProcessStartInfoForSshInternal("", "-- " + remote_command);
}
ProcessStartInfo RemoteUtil::BuildProcessStartInfoForSshPortForward(
int local_port, int remote_port, bool reverse) {
// (internal): Usually, one would pass in -N here, but this makes the
// connection terribly slow! As a workaround, don't use -N (will open a
// shell), but simply eat the output.
ProcessStartInfo si = BuildProcessStartInfoForSshInternal(
GetPortForwardingArg(local_port, remote_port, reverse) + "-n ", "");
si.stdout_handler = [](const void*, size_t) { return absl::OkStatus(); };
return si;
}
ProcessStartInfo RemoteUtil::BuildProcessStartInfoForSshPortForwardAndCommand(
int local_port, int remote_port, bool reverse, std::string remote_command) {
return BuildProcessStartInfoForSshInternal(
GetPortForwardingArg(local_port, remote_port, reverse),
"-- " + remote_command);
}
ProcessStartInfo RemoteUtil::BuildProcessStartInfoForSshInternal(
std::string forward_arg, std::string remote_command_arg) {
ProcessStartInfo start_info;
start_info.command = absl::StrFormat(
"%s "
"%s -tt "
"-F %s "
"-i %s "
"-oServerAliveCountMax=6 " // Number of lost msgs before ssh terminates
"-oServerAliveInterval=5 " // Time interval between alive msgs
"-oStrictHostKeyChecking=yes "
"-oUserKnownHostsFile=\"\"\"%s\"\"\" %s"
"cloudcast@%s -p %i %s",
QuoteArgument(sdk_util_.GetSshExePath()),
quiet_ || verbosity_ < 2 ? "-q" : "",
QuoteArgument(sdk_util_.GetSshConfigPath()),
QuoteArgument(sdk_util_.GetSshKeyFilePath()),
sdk_util_.GetSshKnownHostsFilePath(), forward_arg,
QuoteArgument(gamelet_ip_), ssh_port_, remote_command_arg);
start_info.forward_output_to_log = forward_output_to_log_;
return start_info;
}
absl::Status RemoteUtil::CheckIpPort() {
if (gamelet_ip_.empty() || ssh_port_ == 0) {
return MakeStatus("IP or port not set");
}
return absl::OkStatus();
}
} // namespace cdc_ft

123
common/remote_util.h Normal file
View File

@@ -0,0 +1,123 @@
/*
* Copyright 2022 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef COMMON_REMOTE_UTIL_H_
#define COMMON_REMOTE_UTIL_H_
#include <string>
#include <vector>
#include "absl/status/status.h"
#include "common/process.h"
#include "common/sdk_util.h"
namespace cdc_ft {
// Utilities for executing remote commands on a gamelet through SSH.
// Windows-only.
class RemoteUtil {
public:
// If |verbosity| is > 0 and |quiet| is false, output from scp, ssh etc.
// commands is shown.
// If |quiet| is true, scp, ssh etc. commands use quiet mode.
// If |forward_output_to_log| is true, process output is forwarded to logging
// instead of this process's stdout/stderr.
RemoteUtil(int verbosity, bool quiet, ProcessFactory* process_factory,
bool forward_output_to_log);
// Returns the initialization status. Should be OK unless in case of some rare
// internal error. Should be checked before accessing any members.
const absl::Status& GetInitStatus() const {
return sdk_util_.GetInitStatus();
}
// Set IP of the remote instance and the ssh tunnel port.
void SetIpAndPort(const std::string& gamelet_ip, int ssh_port);
// Copies |source_filepaths| to the remote folder |dest| on the gamelet using
// scp. Must call either InitSsh or SetGameletIp before calling this method.
// If |compress| is true, compressed upload is used.
absl::Status Scp(std::vector<std::string> source_filepaths,
const std::string& dest, bool compress);
// Syncs |source_filepaths| to the remote folder |dest| on the gamelet using
// cdc_rsync. Must call either InitSsh or SetGameletIp before calling this
// method.
absl::Status Sync(std::vector<std::string> source_filepaths,
const std::string& dest);
// Calls 'chmod |mode| |remote_path|' on the gamelet.
// Must call either InitSsh or SetGameletIp before calling this method.
absl::Status Chmod(const std::string& mode, const std::string& remote_path,
bool quiet = false);
// Calls 'rm [-f] |remote_path|' on the gamelet.
// Must call either InitSsh or SetGameletIp before calling this method.
absl::Status Rm(const std::string& remote_path, bool force);
// Calls `mv |old_remote_path| |new_remote_path| on the gamelet.
// Must call either InitSsh or SetGameletIp before calling this method.
absl::Status Mv(const std::string& old_remote_path,
const std::string& new_remote_path);
// Runs |remote_command| on the gamelet. The command must be properly escaped.
// |name| is the name of the command displayed in the logs.
// Must call either InitSsh or SetGameletIp before calling this method.
absl::Status Run(std::string remote_command, std::string name);
// Builds an ssh command that executes |remote_command| on the gamelet.
ProcessStartInfo BuildProcessStartInfoForSsh(std::string remote_command);
// Builds an ssh command that runs SSH port forwarding to the gamelet, using
// the given |local_port| and |remote_port|.
// If |reverse| is true, sets up reverse port forwarding.
// Must call either InitSsh or SetGameletIp before calling this method.
ProcessStartInfo BuildProcessStartInfoForSshPortForward(int local_port,
int remote_port,
bool reverse);
// Builds an ssh command that executes |remote_command| on the gamelet, using
// port forwarding with given |local_port| and |remote_port|.
// If |reverse| is true, sets up reverse port forwarding.
// Must call either InitSsh or SetGameletIp before calling this method.
ProcessStartInfo BuildProcessStartInfoForSshPortForwardAndCommand(
int local_port, int remote_port, bool reverse,
std::string remote_command);
// Returns whether output is suppressed.
bool Quiet() const { return quiet_; }
private:
// Verifies that both |gamelet_ip_| and |ssh_port_| are set.
absl::Status CheckIpPort();
// Common code for BuildProcessStartInfoForSsh*.
ProcessStartInfo BuildProcessStartInfoForSshInternal(
std::string forward_arg, std::string remote_command);
const int verbosity_;
const bool quiet_;
ProcessFactory* const process_factory_;
const bool forward_output_to_log_;
SdkUtil sdk_util_;
std::string gamelet_ip_;
int ssh_port_ = 0;
};
} // namespace cdc_ft
#endif // COMMON_REMOTE_UTIL_H_

106
common/remote_util_test.cc Normal file
View File

@@ -0,0 +1,106 @@
// Copyright 2022 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "common/remote_util.h"
#include "absl/strings/match.h"
#include "common/log.h"
#include "gtest/gtest.h"
namespace cdc_ft {
namespace {
constexpr int kGameletPort = 12345;
constexpr char kGameletPortArg[] = "-p 12345";
constexpr char kGameletIp[] = "1.2.3.4";
constexpr char kGameletIpArg[] = "cloudcast@\"1.2.3.4\"";
constexpr int kLocalPort = 23456;
constexpr int kRemotePort = 34567;
constexpr bool kRegular = false; // Regular port forwarding
constexpr bool kReverse = true; // Reverse port forwarding
constexpr char kPortForwardingArg[] = "-L23456:localhost:34567";
constexpr char kReversePortForwardingArg[] = "-R34567:localhost:23456";
constexpr char kCommand[] = "my_command";
class RemoteUtilTest : public ::testing::Test {
public:
RemoteUtilTest()
: util_(/*verbosity=*/0, /*quiet=*/false, &process_factory_,
/*forward_output_to_log=*/true) {}
void SetUp() override {
Log::Initialize(std::make_unique<ConsoleLog>(LogLevel::kInfo));
util_.SetIpAndPort(kGameletIp, kGameletPort);
}
void TearDown() override { Log::Shutdown(); }
protected:
void ExpectContains(const std::string& str, std::vector<const char*> tokens) {
for (const char* token : tokens) {
EXPECT_TRUE(absl::StrContains(str, token))
<< str << "\ndoes not contain\n"
<< token;
}
}
WinProcessFactory process_factory_;
RemoteUtil util_;
};
TEST_F(RemoteUtilTest, BuildProcessStartInfoForSsh) {
ProcessStartInfo si = util_.BuildProcessStartInfoForSsh(kCommand);
ExpectContains(si.command,
{"ssh.exe", "GGP\\ssh\\id", "oStrictHostKeyChecking=yes",
"oUserKnownHostsFile", "known_hosts", kGameletPortArg,
kGameletIpArg, kCommand});
}
TEST_F(RemoteUtilTest, BuildProcessStartInfoForSshPortForward) {
ProcessStartInfo si = util_.BuildProcessStartInfoForSshPortForward(
kLocalPort, kRemotePort, kRegular);
ExpectContains(si.command,
{"ssh.exe", "GGP\\ssh\\id", "oStrictHostKeyChecking=yes",
"oUserKnownHostsFile", "known_hosts", kGameletPortArg,
kGameletIpArg, kPortForwardingArg});
si = util_.BuildProcessStartInfoForSshPortForward(kLocalPort, kRemotePort,
kReverse);
ExpectContains(si.command,
{"ssh.exe", "GGP\\ssh\\id", "oStrictHostKeyChecking=yes",
"oUserKnownHostsFile", "known_hosts", kGameletPortArg,
kGameletIpArg, kReversePortForwardingArg});
}
TEST_F(RemoteUtilTest, BuildProcessStartInfoForSshPortForwardAndCommand) {
ProcessStartInfo si = util_.BuildProcessStartInfoForSshPortForwardAndCommand(
kLocalPort, kRemotePort, kRegular, kCommand);
ExpectContains(si.command,
{"ssh.exe", "GGP\\ssh\\id", "oStrictHostKeyChecking=yes",
"oUserKnownHostsFile", "known_hosts", kGameletPortArg,
kGameletIpArg, kPortForwardingArg, kCommand});
si = util_.BuildProcessStartInfoForSshPortForwardAndCommand(
kLocalPort, kRemotePort, kReverse, kCommand);
ExpectContains(si.command,
{"ssh.exe", "GGP\\ssh\\id", "oStrictHostKeyChecking=yes",
"oUserKnownHostsFile", "known_hosts", kGameletPortArg,
kGameletIpArg, kReversePortForwardingArg, kCommand});
}
} // namespace
} // namespace cdc_ft

77
common/scoped_handle_win.cc Normal file
View File

@@ -0,0 +1,77 @@
// Copyright 2022 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "common/scoped_handle_win.h"
#define WIN32_LEAN_AND_MEAN
#include <windows.h>
#include <cassert>
#include <memory>
namespace cdc_ft {
ScopedHandle::ScopedHandle() : handle_(nullptr) {}
ScopedHandle::ScopedHandle(HANDLE handle) : handle_(nullptr) { Set(handle); }
ScopedHandle::ScopedHandle(ScopedHandle&& other) : handle_(nullptr) {
Set(other.Release());
}
ScopedHandle::~ScopedHandle() { Close(); }
bool ScopedHandle::IsValid() const { return IsHandleValid(handle_); }
ScopedHandle& ScopedHandle::operator=(ScopedHandle&& other) {
Set(other.Release());
return *this;
}
void ScopedHandle::Set(HANDLE handle) {
if (handle_ != handle) {
// Preserve old error code.
uint32_t last_error = GetLastError();
Close();
if (IsHandleValid(handle)) {
handle_ = handle;
}
SetLastError(last_error);
}
}
HANDLE ScopedHandle::Get() const {
assert(IsValid());
return handle_;
}
HANDLE ScopedHandle::Release() {
HANDLE temp = handle_;
handle_ = nullptr;
return temp;
}
void ScopedHandle::Close() {
if (IsHandleValid(handle_)) {
CloseHandle(handle_);
handle_ = nullptr;
}
}
bool ScopedHandle::IsHandleValid(HANDLE handle) {
return handle != nullptr && handle != INVALID_HANDLE_VALUE;
}
} // namespace cdc_ft

61
common/scoped_handle_win.h Normal file
View File

@@ -0,0 +1,61 @@
/*
* Copyright 2022 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef COMMON_SCOPED_HANDLE_WIN_H_
#define COMMON_SCOPED_HANDLE_WIN_H_
using HANDLE = void*;
namespace cdc_ft {
// In Windows, some functions return nullptr as invalid handles, some
// INVALID_HANDLE_VALUE. This class attempts to minimize this pain.
class ScopedHandle {
public:
ScopedHandle();
explicit ScopedHandle(HANDLE handle);
ScopedHandle(ScopedHandle&& other);
~ScopedHandle();
ScopedHandle(const ScopedHandle&) = delete;
ScopedHandle& operator=(const ScopedHandle&) = delete;
bool IsValid() const;
ScopedHandle& operator=(ScopedHandle&& other);
void Set(HANDLE handle);
HANDLE Get() const;
// Transfers ownership away from this object.
HANDLE Release();
// Explicitly closes the owned handle.
void Close();
private:
static bool IsHandleValid(HANDLE handle);
HANDLE handle_;
};
} // namespace cdc_ft
#endif // COMMON_SCOPED_HANDLE_WIN_H_

93
common/sdk_util.cc Normal file
View File

@@ -0,0 +1,93 @@
// Copyright 2022 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "common/sdk_util.h"
#include <cassert>
#include <string>
#include "common/clock.h"
#include "common/path.h"
#include "common/status_macros.h"
namespace cdc_ft {
SdkUtil::SdkUtil() {
init_status_ = path::GetKnownFolderPath(path::FolderId::kRoamingAppData,
&roaming_appdata_path_);
init_status_.Update(path::GetKnownFolderPath(path::FolderId::kProgramFiles,
&program_files_path_));
absl::Status status = path::GetEnv("GGP_SDK_PATH", &ggp_sdk_path_env_);
if (absl::IsNotFound(status) || ggp_sdk_path_env_.empty())
ggp_sdk_path_env_ = path::Join(program_files_path_, "GGP SDK");
// Create an empty config file if it does not exist yet.
const std::string ssh_config_path = GetSshConfigPath();
if (!path::Exists(ssh_config_path)) {
init_status_.Update(path::CreateDirRec(path::DirName(ssh_config_path)));
init_status_.Update(path::WriteFile(ssh_config_path, nullptr, 0));
}
}
SdkUtil::~SdkUtil() = default;
std::string SdkUtil::GetUserConfigPath() const {
assert(init_status_.ok());
return path::Join(roaming_appdata_path_, "GGP");
}
std::string SdkUtil::GetServicesConfigPath() const {
return path::Join(GetUserConfigPath(), "services");
}
std::string SdkUtil::GetLogPath(const char* log_base_name) const {
DefaultSystemClock clock;
std::string timestamp_ext = clock.FormatNow(".%Y%m%d-%H%M%S.log", false);
return path::Join(GetUserConfigPath(), "logs", log_base_name + timestamp_ext);
}
std::string SdkUtil::GetSshConfigPath() const {
return path::Join(GetUserConfigPath(), "ssh", "config");
}
std::string SdkUtil::GetSshKeyFilePath() const {
return path::Join(GetUserConfigPath(), "ssh", "id_rsa");
}
std::string SdkUtil::GetSshKnownHostsFilePath() const {
return path::Join(GetUserConfigPath(), "ssh", "known_hosts");
}
std::string SdkUtil::GetSDKPath() const {
assert(init_status_.ok());
return ggp_sdk_path_env_;
}
std::string SdkUtil::GetDevBinPath() const {
return path::Join(GetSDKPath(), "dev", "bin");
}
std::string SdkUtil::GetSshPath() const {
return path::Join(GetSDKPath(), "tools", "OpenSSH-Win64");
}
std::string SdkUtil::GetSshExePath() const {
return path::Join(GetSshPath(), "ssh.exe");
}
std::string SdkUtil::GetScpExePath() const {
return path::Join(GetSshPath(), "scp.exe");
}
} // namespace cdc_ft

96
common/sdk_util.h Normal file
View File

@@ -0,0 +1,96 @@
/*
* Copyright 2022 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef COMMON_SDK_UTIL_H_
#define COMMON_SDK_UTIL_H_
#include <string>
#include "absl/status/status.h"
#include "absl/status/statusor.h"
#include "common/platform.h"
#if !PLATFORM_WINDOWS
#error SdkUtil only supports Windows so far.
#endif
namespace cdc_ft {
// Provides paths to selected files in the Stadia Windows SDK.
class SdkUtil {
public:
SdkUtil();
~SdkUtil();
// Returns the initialization status. Should be OK unless in case of some rare
// internal error. Should be checked before accessing any members.
const absl::Status& GetInitStatus() const { return init_status_; }
// Returns the path of the SDK user configuration, e.g.
// %APPDATA%\GGP.
std::string GetUserConfigPath() const;
// Returns the path of the SDK services configuration, e.g.
// %APPDATA%\GGP\services.
std::string GetServicesConfigPath() const;
// Returns the path of a log file with given |log_base_name|, e.g.
// %APPDATA%\GGP\logs\log_base_name.20210729-125930.log.
std::string GetLogPath(const char* log_base_name) const;
// Returns the path of the ssh configuration file, e.g.
// %APPDATA%\GGP\ssh\config.
std::string GetSshConfigPath() const;
// Returns the path of the ssh private key file in the SDK configuration, e.g.
// %APPDATA%\GGP\ssh\id_rsa.
std::string GetSshKeyFilePath() const;
// Returns the path of the ssh known_hosts file in the SDK configuration, e.g.
// %APPDATA%\GGP\ssh\known_hosts.
std::string GetSshKnownHostsFilePath() const;
// Returns the path of the installed SDK, e.g.
// C:\Program Files\GGP SDK.
std::string GetSDKPath() const;
// Returns the path of the dev tools that ship with the SDK, e.g.
// C:\Program Files\GGP SDK\dev\bin.
std::string GetDevBinPath() const;
// Returns the path of the OpenSSH tools that ship with the SDK, e.g.
// C:\Program Files\GGP SDK\tools\OpenSSH-Win64.
std::string GetSshPath() const;
// Returns the path of ssh.exe that ships with the SDK, e.g.
// C:\Program Files\GGP SDK\tools\OpenSSH-Win64\ssh.exe.
std::string GetSshExePath() const;
// Returns the path of scp.exe that ships with the SDK, e.g.
// C:\Program Files\GGP SDK\tools\OpenSSH-Win64\scp.exe.
std::string GetScpExePath() const;
private:
std::string roaming_appdata_path_;
std::string program_files_path_;
std::string ggp_sdk_path_env_;
absl::Status init_status_;
std::string full_sdk_version_;
};
} // namespace cdc_ft
#endif // COMMON_SDK_UTIL_H_

115
common/sdk_util_test.cc Normal file
View File

@@ -0,0 +1,115 @@
// Copyright 2022 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "common/sdk_util.h"
#include <stdlib.h>
#include "common/log.h"
#include "common/path.h"
#include "common/status_test_macros.h"
#include "gtest/gtest.h"
namespace cdc_ft {
namespace {
constexpr bool kCreateFile = true;
constexpr bool kDontCreateFile = false;
class SdkUtilTest : public ::testing::Test {
public:
void SetUp() override {
Log::Initialize(std::make_unique<ConsoleLog>(LogLevel::kInfo));
}
void TearDown() override {
Log::Shutdown();
for (std::string dir_path : test_created_directories_) {
EXPECT_OK(path::RemoveDirRec(dir_path));
}
}
protected:
void CheckSdkPaths(const SdkUtil& sdk_util, const std::string& sdk_dir) {
EXPECT_EQ(sdk_util.GetSDKPath(), sdk_dir);
EXPECT_EQ(sdk_util.GetSshPath(),
path::Join(sdk_dir, "tools\\OpenSSH-Win64"));
EXPECT_EQ(sdk_util.GetSshExePath(),
path::Join(sdk_dir, "tools\\OpenSSH-Win64\\ssh.exe"));
EXPECT_EQ(sdk_util.GetScpExePath(),
path::Join(sdk_dir, "tools\\OpenSSH-Win64\\scp.exe"));
EXPECT_EQ(sdk_util.GetDevBinPath(), path::Join(sdk_dir, "dev", "bin"));
}
void SetupGetSdkVersion(const std::string& file_content,
bool create_version_file) {
std::string ggp_sdk_path = std::tmpnam(nullptr);
EXPECT_OK(path::SetEnv("GGP_SDK_PATH", ggp_sdk_path));
test_created_directories_.push_back(ggp_sdk_path);
EXPECT_OK(path::CreateDirRec(ggp_sdk_path));
if (create_version_file) {
std::string version_path = path::Join(ggp_sdk_path, "VERSION");
EXPECT_OK(path::WriteFile(version_path, file_content.c_str(),
file_content.size()));
}
}
// Contains assets which where created during test.
// The assets are to be deleted in the end of the test.
std::vector<std::string> test_created_directories_;
};
TEST_F(SdkUtilTest, CheckRoamingAppDataPaths) {
SdkUtil sdk_util;
EXPECT_OK(sdk_util.GetInitStatus());
std::string appdata_dir;
EXPECT_OK(
path::GetKnownFolderPath(path::FolderId::kRoamingAppData, &appdata_dir));
const std::string ggp_path = path::Join(appdata_dir, "GGP");
EXPECT_EQ(sdk_util.GetUserConfigPath(), ggp_path);
EXPECT_EQ(sdk_util.GetServicesConfigPath(), path::Join(ggp_path, "services"));
EXPECT_EQ(sdk_util.GetSshConfigPath(), path::Join(ggp_path, "ssh", "config"));
EXPECT_EQ(sdk_util.GetSshKeyFilePath(),
path::Join(ggp_path, "ssh", "id_rsa"));
EXPECT_EQ(sdk_util.GetSshKnownHostsFilePath(),
path::Join(ggp_path, "ssh", "known_hosts"));
}
TEST_F(SdkUtilTest, CheckSdkPathsWithoutGgpSdkPathEnv) {
// Clear environment variable and figure out default SDK dir.
EXPECT_OK(path::SetEnv("GGP_SDK_PATH", ""));
std::string program_files_dir;
EXPECT_OK(path::GetKnownFolderPath(path::FolderId::kProgramFiles,
&program_files_dir));
const std::string sdk_dir = path::Join(program_files_dir, "GGP SDK");
SdkUtil sdk_util;
EXPECT_OK(sdk_util.GetInitStatus());
CheckSdkPaths(sdk_util, sdk_dir);
}
TEST_F(SdkUtilTest, CheckSdkPathsWithGgpSdkPathEnv) {
// Set a path with unicode character.
std::string sdk_dir = u8"C:\\I\\♥\\GGP SDK\\";
EXPECT_OK(path::SetEnv("GGP_SDK_PATH", sdk_dir));
SdkUtil sdk_util;
EXPECT_OK(sdk_util.GetInitStatus());
CheckSdkPaths(sdk_util, sdk_dir);
}
} // namespace
} // namespace cdc_ft

35
common/semaphore.cc Normal file
View File

@@ -0,0 +1,35 @@
// Copyright 2022 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "common/semaphore.h"
namespace cdc_ft {
Semaphore::Semaphore(int initial_count) : count_(initial_count) {}
Semaphore::~Semaphore() = default;
void Semaphore::Wait() {
std::unique_lock<std::mutex> lock(mutex_);
condition_.wait(lock, [&]() { return count_ > 0; });
count_--;
}
void Semaphore::Signal() {
std::lock_guard<std::mutex> lock(mutex_);
count_++;
condition_.notify_one();
}
} // namespace cdc_ft

52
common/semaphore.h Normal file
View File

@@ -0,0 +1,52 @@
/*
* Copyright 2022 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef COMMON_SEMAPHORE_H_
#define COMMON_SEMAPHORE_H_
#include <condition_variable>
#include <mutex>
namespace cdc_ft {
// Standard semaphore. Can be replaced by std::counting_semaphore in C++20.
// Not fair! Wait() does not unlock in FIFO order.
class Semaphore {
public:
// Initialize the semaphore with |initial_count|. The count indicates how
// often Wait() can be called without blocking. There is no max count.
explicit Semaphore(int initial_count);
~Semaphore();
// If the count is larger than zero, decreases the count and returns
// immediately. If the count is zero, blocks until some other thread calls
// Signal() and decreases the count.
// This method is thread-safe.
void Wait();
// Increases the count.
// This method is thread-safe.
void Signal();
private:
uint32_t count_;
std::mutex mutex_;
std::condition_variable condition_;
};
} // namespace cdc_ft
#endif // COMMON_SEMAPHORE_H_

122
common/semaphore_test.cc Normal file
View File

@@ -0,0 +1,122 @@
// Copyright 2022 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "common/semaphore.h"
#include <atomic>
#include <thread>
#include <vector>
#include "common/util.h"
#include "gtest/gtest.h"
namespace cdc_ft {
namespace {
class SemaphoreTest : public ::testing::Test {
protected:
bool PollUntil(std::function<bool()> predicate, uint32_t timeout_ms) {
uint32_t ms = 0;
while (!predicate()) {
Util::Sleep(1);
ms++;
if (ms >= timeout_ms) {
return false;
}
}
return true;
}
};
TEST_F(SemaphoreTest, BlocksIfInitialCountIsZero) {
Semaphore semaphore(0);
std::atomic_bool ready(false);
std::atomic_bool done(false);
std::thread thread([&semaphore, &ready, &done]() {
ready = true;
semaphore.Wait();
done = true;
});
EXPECT_TRUE(PollUntil([&ready]() { return ready.load(); }, 5000));
// This will time out, so use a short timeout. Is there a way to test whether
// the thread is blocking without using a timeout?
EXPECT_FALSE(PollUntil([&done]() { return done.load(); }, 10));
semaphore.Signal();
EXPECT_TRUE(PollUntil([&done]() { return done.load(); }, 5000));
thread.join();
}
TEST_F(SemaphoreTest, DoesNotBlockIfInitialCountIsOne) {
Semaphore semaphore(1);
std::atomic_bool first_wait(false);
std::atomic_bool done(false);
std::thread thread([&semaphore, &first_wait, &done]() {
semaphore.Wait();
first_wait = true;
semaphore.Wait();
done = true;
});
EXPECT_TRUE(PollUntil([&first_wait]() { return first_wait.load(); }, 5000));
// This will time out, so use a short timeout. Is there a way to test whether
// the thread is blocking without using a timeout?
EXPECT_FALSE(PollUntil([&done]() { return done.load(); }, 10));
semaphore.Signal();
EXPECT_TRUE(PollUntil([&done]() { return done.load(); }, 5000));
thread.join();
}
TEST_F(SemaphoreTest, SignalManyThreads) {
Semaphore semaphore(0);
std::atomic_int a(0);
std::atomic_int b(0);
std::atomic_int c(0);
const int N = 16;
std::vector<std::thread> threads;
for (int n = 0; n < N; ++n) {
threads.emplace_back([&semaphore, &a, &b, &c]() {
++a;
semaphore.Wait();
++b;
semaphore.Wait();
++c;
});
}
// All threads should be at the first wait.
EXPECT_TRUE(PollUntil([&]() { return a == N; }, 5000));
for (int n = 0; n < N; ++n) {
semaphore.Signal();
}
// Some threads should be at or past the second wait.
// Note: If the Semaphore were fair, it would be cnt[1] == N and cnt[2] == 0.
EXPECT_TRUE(PollUntil([&]() { return b + c == N; }, 5000));
for (int n = 0; n < N; ++n) {
semaphore.Signal();
}
// All threads should have finished.
EXPECT_TRUE(PollUntil([&]() { return b == N && c == N; }, 5000));
for (int n = 0; n < N; ++n) {
threads[n].join();
}
}
} // namespace
} // namespace cdc_ft

158
common/stats_collector.cc Normal file
View File

@@ -0,0 +1,158 @@
// Copyright 2022 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "common/stats_collector.h"
#include <map>
#include <thread>
#include "absl/strings/str_format.h"
#include "absl/synchronization/mutex.h"
#include "common/log.h"
#include "common/stopwatch.h"
namespace cdc_ft {
namespace {
constexpr absl::Duration kStatsPrintPeriod = absl::Seconds(1);
}
class StatsCollectorImpl : public StatsCollector {
public:
class DurationScopeImpl : public DurationScope {
public:
DurationScopeImpl(StatsCollectorImpl* stats_collector, const char* name)
: stats_collector_(stats_collector), name_(name) {
assert(stats_collector_);
}
~DurationScopeImpl() override {
stats_collector_->DoRecordDuration(name_, stopwatch_.Elapsed());
}
private:
StatsCollectorImpl* stats_collector_;
const char* name_;
Stopwatch stopwatch_;
};
StatsCollectorImpl() {
print_thread_ = std::thread([this]() { ThreadPrintMain(); });
}
~StatsCollectorImpl() override {
{
absl::MutexLock lock(&mutex_);
shutdown_ = true;
}
if (print_thread_.joinable()) {
print_thread_.join();
}
}
void IncCounter(const char* name, size_t value) override
ABSL_LOCKS_EXCLUDED(mutex_) {
absl::MutexLock lock(&mutex_);
counters_[name] += value;
}
std::unique_ptr<DurationScope> RecordDuration(const char* name) override {
return std::make_unique<DurationScopeImpl>(this, name);
}
private:
void DoRecordDuration(const char* name, absl::Duration duration)
ABSL_LOCKS_EXCLUDED(mutex_) {
absl::MutexLock lock(&mutex_);
durations_[std::move(name)] += duration;
}
void ThreadPrintMain() ABSL_LOCKS_EXCLUDED(mutex_) {
absl::MutexLock lock(&mutex_);
while (!shutdown_) {
Stopwatch sw;
auto cond = [&]() ABSL_EXCLUSIVE_LOCKS_REQUIRED(mutex_) {
return shutdown_ || sw.Elapsed() >= kStatsPrintPeriod;
};
mutex_.Await(absl::Condition(&cond));
if (shutdown_) {
break;
}
if (counters_.empty() && durations_.empty()) continue;
std::string line;
for (const auto& [name, value] : counters_) {
line += absl::StrFormat("%s:%u ", name, value);
}
for (const auto& [name, value] : durations_) {
line +=
absl::StrFormat("%s:%0.3fs ", name, absl::ToDoubleSeconds(value));
}
counters_.clear();
durations_.clear();
LOG_WARNING("\n%s\n", line);
}
}
absl::Mutex mutex_;
std::map<std::string, size_t> counters_ ABSL_GUARDED_BY(mutex_);
std::map<std::string, absl::Duration> durations_ ABSL_GUARDED_BY(mutex_);
std::thread print_thread_;
bool shutdown_ ABSL_GUARDED_BY(mutex_) = false;
};
class NullStatsCollector : public StatsCollector {
public:
class NullDurationScope : public DurationScope {};
NullStatsCollector() = default;
~NullStatsCollector() override = default;
void IncCounter(const char*, size_t) override {}
std::unique_ptr<DurationScope> RecordDuration(const char*) override {
return std::make_unique<NullDurationScope>();
}
};
StatsCollector::StatsCollector() = default;
StatsCollector::~StatsCollector() = default;
StatsCollector* StatsCollector::instance_ = nullptr;
StatsCollector::DurationScope::~DurationScope() = default;
// static
void StatsCollector::Initialize() {
assert(!instance_);
instance_ = new StatsCollectorImpl;
}
// static
void StatsCollector::Shutdown() {
assert(instance_);
delete instance_;
instance_ = nullptr;
}
// static
StatsCollector* StatsCollector::Instance() {
static NullStatsCollector null_instance;
return instance_ ? instance_ : &null_instance;
}
} // namespace cdc_ft

54
common/stats_collector.h Normal file
View File

@@ -0,0 +1,54 @@
/*
* Copyright 2022 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef COMMON_STATS_COLLECTOR_H_
#define COMMON_STATS_COLLECTOR_H_
#include <memory>
namespace cdc_ft {
class StatsCollector {
public:
StatsCollector();
virtual ~StatsCollector();
// Initializes the stats collector. If not called, the instance returned from
// Instance() is a no-op stats collector where all calls have no effect.
static void Initialize();
// Shuts down the stats collector. Should be called during shutdown if
// Initialize() was called during startup.
static void Shutdown();
// Returns a no-op stats collector if Initialize() has not been called.
static StatsCollector* Instance();
class DurationScope {
public:
virtual ~DurationScope();
};
virtual void IncCounter(const char* name, size_t value = 1) = 0;
virtual std::unique_ptr<DurationScope> RecordDuration(const char* name) = 0;
private:
static StatsCollector* instance_;
};
} // namespace cdc_ft
#endif // COMMON_STATS_COLLECTOR_H_

54
common/status.cc Normal file
View File

@@ -0,0 +1,54 @@
// Copyright 2022 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "common/status.h"
#include "absl/strings/numbers.h"
namespace cdc_ft {
namespace {
constexpr char kTagKey[] = "tag";
absl::Cord TagToCord(Tag tag) {
return absl::Cord(std::to_string(static_cast<int>(tag)));
}
absl::optional<Tag> CordToTag(absl::optional<absl::Cord> cord) {
if (!cord.has_value()) return {};
int value;
absl::numbers_internal::safe_strto32_base(std::string(cord.value()), &value,
10);
return value >= 0 && value < static_cast<int>(Tag::kCount)
? static_cast<Tag>(value)
: absl::optional<Tag>();
}
} // namespace
absl::Status SetTag(absl::Status status, Tag tag) {
status.SetPayload(kTagKey, TagToCord(tag));
return status;
}
bool HasTag(const absl::Status& status, Tag tag) {
return status.GetPayload(kTagKey) == TagToCord(tag);
}
absl::optional<Tag> GetTag(const absl::Status& status) {
return CordToTag(status.GetPayload(kTagKey));
}
} // namespace cdc_ft

101
common/status.h Normal file
View File

@@ -0,0 +1,101 @@
/*
* Copyright 2022 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef COMMON_STATUS_H_
#define COMMON_STATUS_H_
#include "absl/status/status.h"
#include "absl/strings/str_cat.h"
#include "absl/strings/str_format.h"
#include "common/platform.h"
namespace cdc_ft {
//
// Convenience helper functions for status creation.
//
inline absl::Status MakeStatus(const char* message) {
return absl::InternalError(message);
}
template <typename... Args>
absl::Status MakeStatus(const absl::FormatSpec<Args...>& format, Args... args) {
return absl::InternalError(absl::StrFormat(format, args...));
}
// Convenience helper for cases that may or may not have a message to wrap.
template <typename... Args>
absl::Status WrapStatus(absl::Status inner_status) {
return inner_status;
}
// Returns OK if |inner_status| is OK and a copy of |inner_status| with given
// message + |inner_status|'s message otherwise.
template <typename... Args>
absl::Status WrapStatus(absl::Status inner_status,
const absl::FormatSpec<Args...>& format, Args... args) {
if (inner_status.ok()) {
return inner_status;
}
std::string message = absl::StrFormat(format, args...);
absl::Status wrapped_status(
inner_status.code(), absl::StrCat(inner_status.message(), "; ", message));
inner_status.ForEachPayload(
[&wrapped_status](absl::string_view key, const absl::Cord& value) {
wrapped_status.SetPayload(key, value);
});
return wrapped_status;
}
//
// Tags attached to absl::Status to allow more fine-grained error handling.
//
enum class Tag : uint8_t {
// Sending end of the socket was closed, so receiving end can no longer read.
kSocketEof = 0,
// Bind failed, probably because there's another instance of cdc_rsync
// running.
kAddressInUse = 1,
// The gamelet components need to be re-deployed.
kDeployServer = 2,
// Something asks for user input, but we're in quiet mode.
kInstancePickerNotAvailableInQuietMode = 3,
// Timeout while trying to connect to the gamelet component.
kConnectionTimeout = 4,
// MUST BE LAST.
kCount = 5,
};
// Tags a status. No-op if |status| is OK. Overwrites existing tags.
absl::Status SetTag(absl::Status status, Tag tag);
// Returns the associated tag. Empty if none.
absl::optional<Tag> GetTag(const absl::Status& status);
// Checks whether the status has a certain tag.
bool HasTag(const absl::Status& status, Tag tag);
} // namespace cdc_ft
#endif // COMMON_STATUS_H_

42
common/status_macros.h Normal file
View File

@@ -0,0 +1,42 @@
/*
* Copyright 2022 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef COMMON_STATUS_MACROS_H_
#define COMMON_STATUS_MACROS_H_
#include "absl/status/status.h"
#include "absl/status/statusor.h"
#include "absl/strings/str_format.h"
#include "common/status.h"
#define RETURN_IF_ERROR(expr, ...) \
do { \
const absl::Status __status__ = (expr); \
if (!__status__.ok()) { \
return ::cdc_ft::WrapStatus(__status__, ##__VA_ARGS__); \
} \
} while (0)
#define ASSIGN_OR_RETURN(lhs, expr, ...) \
do { \
auto __status__ = (expr); \
if (!__status__.ok()) { \
return ::cdc_ft::WrapStatus(__status__.status(), ##__VA_ARGS__); \
} \
lhs = std::move(__status__.value()); \
} while (0)
#endif // COMMON_STATUS_MACROS_H_

82
common/status_test_macros.h Normal file
View File

@@ -0,0 +1,82 @@
/*
* Copyright 2022 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef COMMON_STATUS_TEST_MACROS_H_
#define COMMON_STATUS_TEST_MACROS_H_
#include "absl/status/status.h"
#include "absl/status/statusor.h"
#include "common/platform.h"
#ifndef PLATFORM_WINDOWS
#define __forceinline inline __attribute__((always_inline))
#endif
namespace internal {
namespace status_test_macros {
// Helper functions to get status from both status and StatusOr.
template <class T>
const absl::Status& ToStatus(const absl::StatusOr<T>& result) {
return result.status();
}
__forceinline const absl::Status& ToStatus(const absl::Status& status) {
return status;
}
} // namespace status_test_macros
} // namespace internal
// Little convenience macros to check the return value of an absl::Status in
// gTest unit tests.
#define EXPECT_OK(x) VERIFY_OK(EXPECT, x)
#define ASSERT_OK(x) VERIFY_OK(ASSERT, x)
#define VERIFY_OK(action, x) \
do { \
auto __status = ::internal::status_test_macros::ToStatus(x); \
action##_TRUE(__status.ok()) << "Error: " << __status.ToString(); \
} while (0)
#define EXPECT_NOT_OK(x) VERIFY_NOT_OK(EXPECT, x)
#define ASSERT_NOT_OK(x) VERIFY_NOT_OK(ASSERT, x)
#define VERIFY_NOT_OK(action, x) action##_FALSE((x).ok())
// type should be one of the absl::IsXXX errors, e.g. "Canceled".
#define EXPECT_ERROR(type, x) VERIFY_ERROR(EXPECT, type, x)
#define ASSERT_ERROR(type, x) VERIFY_ERROR(ASSERT, type, x)
#define VERIFY_ERROR(action, type, x) \
do { \
auto __status = ::internal::status_test_macros::ToStatus(x); \
action##_TRUE(absl::Is##type(__status)) \
<< "Unexpected status '" << __status.ToString() << "', expected " \
<< #type; \
} while (0)
// type should be one of the absl::IsXXX errors, e.g. "Canceled".
#define EXPECT_ERROR_MSG(type, msg, x) VERIFY_ERROR_MSG(EXPECT, type, msg, x)
#define ASSERT_ERROR_MSG(type, msg, x) VERIFY_ERROR_MSG(ASSERT, type, msg, x)
#define VERIFY_ERROR_MSG(action, type, msg, x) \
do { \
auto __status = ::internal::status_test_macros::ToStatus(x); \
action##_TRUE(absl::Is##type(__status)) \
<< "Unexpected status '" << __status.ToString() << "', expected " \
<< #type; \
action##_NE(__status.message().find(msg), std::string::npos) \
<< "Unexpected message '" << __status.message() << "', expected " \
<< "'" << msg << "'"; \
} while (0)
#endif // COMMON_STATUS_TEST_MACROS_H_

35
common/stopwatch.cc Normal file
View File

@@ -0,0 +1,35 @@
// Copyright 2022 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "common/stopwatch.h"
namespace cdc_ft {
Stopwatch::Stopwatch(SteadyClock* clock) : clock_(clock) { Reset(); }
Stopwatch::~Stopwatch() = default;
double Stopwatch::ElapsedSeconds() const {
// Assigning to a duration<double> automagically converts to seconds.
const std::chrono::duration<double> elapsed = clock_->Now() - start_;
return elapsed.count();
}
void Stopwatch::Reset() { start_ = clock_->Now(); }
absl::Duration Stopwatch::Elapsed() const {
return absl::FromChrono(clock_->Now() - start_);
}
} // namespace cdc_ft

52
common/stopwatch.h Normal file
View File

@@ -0,0 +1,52 @@
/*
* Copyright 2022 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef COMMON_STOPWATCH_H_
#define COMMON_STOPWATCH_H_
#include <chrono>
#include "absl/time/time.h"
#include "common/clock.h"
namespace cdc_ft {
class Stopwatch {
public:
using Timestamp = SteadyClock::Timestamp;
explicit Stopwatch(SteadyClock* clock = DefaultSteadyClock::GetInstance());
~Stopwatch();
// Returns the elapsed time (in seconds) relative to construction or to the
// last time when Reset() was called.
double ElapsedSeconds() const;
// Resets the elapsed time to zero.
void Reset();
// Returns the duration relative to construction or to the last time when
// Reset() was called.
absl::Duration Elapsed() const;
private:
SteadyClock* clock_;
Timestamp start_;
};
} // namespace cdc_ft
#endif // COMMON_STOPWATCH_H_

47
common/stopwatch_test.cc Normal file
View File

@@ -0,0 +1,47 @@
// Copyright 2022 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "common/stopwatch.h"
#include "common/testing_clock.h"
#include "gtest/gtest.h"
namespace cdc_ft {
namespace {
class StopwatchTest : public ::testing::Test {};
TEST_F(StopwatchTest, DefaultStopwatch) {
// Cover DefaultClock.
Stopwatch stopwatch;
EXPECT_GE(stopwatch.ElapsedSeconds(), 0.0);
}
TEST_F(StopwatchTest, Stopwatch) {
TestingSteadyClock clock;
Stopwatch stopwatch(&clock);
EXPECT_EQ(stopwatch.ElapsedSeconds(), 0.0);
clock.Advance(1000);
EXPECT_EQ(stopwatch.ElapsedSeconds(), 1.0);
clock.Advance(500);
EXPECT_EQ(stopwatch.ElapsedSeconds(), 1.5);
stopwatch.Reset();
EXPECT_EQ(stopwatch.ElapsedSeconds(), 0.0);
clock.Advance(250);
EXPECT_EQ(stopwatch.ElapsedSeconds(), 0.25);
}
} // namespace
} // namespace cdc_ft

158
common/stub_process.cc Normal file
View File

@@ -0,0 +1,158 @@
// Copyright 2022 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "common/stub_process.h"
#include "absl/strings/match.h"
#include "absl/strings/str_format.h"
#include "common/status_macros.h"
#include "common/util.h"
namespace cdc_ft {
class StubProcess : public Process {
public:
StubProcess(const ProcessStartInfo& start_info, std::string std_out,
std::string std_err, uint32_t exit_code, bool has_exited,
bool never_exits, StubProcessFactory::ProcessHandlerFn handler)
: Process(start_info),
std_out_(std::move(std_out)),
std_err_(std::move(std_err)),
exit_code_(exit_code),
has_exited_(has_exited),
never_exits_(never_exits),
handler_(handler) {}
~StubProcess() = default;
// Process:
absl::Status Start() override { return absl::OkStatus(); }
absl::Status RunUntil(std::function<bool()> exit_condition) override {
if (handler_) {
handler_(&std_out_, &std_err_, &exit_code_);
}
// Write stdout/stderr including the null terminator.
if (start_info_.stdout_handler) {
RETURN_IF_ERROR(
start_info_.stdout_handler(std_out_.c_str(), std_out_.size() + 1));
}
if (start_info_.stderr_handler) {
RETURN_IF_ERROR(
start_info_.stderr_handler(std_out_.c_str(), std_err_.size() + 1));
}
if (never_exits_) {
// Poll until exit condition (e.g. timeout) is satisfied.
while (!exit_condition()) Util::Sleep(1);
} else {
// Exit immediately.
has_exited_ = true;
}
return absl::OkStatus();
}
absl::Status Terminate() override { return absl::OkStatus(); }
absl::Status WriteToStdIn(const void*, size_t) override {
return absl::OkStatus();
}
void CloseStdIn() override {}
bool HasExited() const override { return has_exited_; }
uint32_t ExitCode() const override { return exit_code_; }
absl::Status GetStatus() const override { return absl::OkStatus(); }
private:
std::string std_out_;
std::string std_err_;
uint32_t exit_code_;
const bool never_exits_;
bool has_exited_ = false;
StubProcessFactory::ProcessHandlerFn handler_;
};
// An ErrorProcess is returned if the corresponding command isn't handled in
// StubProcessFactory::Create(). It will fail to start.
class ErrorProcess : public Process {
public:
explicit ErrorProcess(const ProcessStartInfo& start_info)
: Process(start_info),
status_(absl::InternalError(
absl::StrFormat("Unhandled process '%s'", start_info_.command))) {}
~ErrorProcess() = default;
// Process:
absl::Status Start() override { return status_; }
absl::Status RunUntil(std::function<bool()>) override { return status_; }
absl::Status Terminate() override { return status_; }
absl::Status WriteToStdIn(const void*, size_t) override { return status_; }
void CloseStdIn() override {}
bool HasExited() const override { return false; }
uint32_t ExitCode() const override { return kExitCodeStillRunning; }
absl::Status GetStatus() const override { return status_; }
private:
absl::Status status_;
};
StubProcessFactory::~StubProcessFactory() = default;
std::unique_ptr<Process> StubProcessFactory::Create(
const ProcessStartInfo& start_info) {
for (const ProcessInfo& pi : process_info_) {
if (absl::StrContains(start_info.command, pi.command_part)) {
return std::make_unique<StubProcess>(start_info, pi.std_out, pi.std_err,
pi.exit_code, pi.has_exited,
pi.never_exits, pi.handler);
}
}
return std::make_unique<ErrorProcess>(start_info);
}
void StubProcessFactory::SetProcessOutput(std::string command_part,
std::string std_out,
std::string std_err,
uint32_t exit_code) {
process_info_.push_back({std::move(command_part), std::move(std_out),
std::move(std_err), exit_code,
/*has_exited=*/false, /*never_exits=*/false,
ProcessHandlerFn()});
}
void StubProcessFactory::SetProcessNeverExits(std::string command_part) {
process_info_.push_back({std::move(command_part), "", "", 1,
/*has_exited=*/false,
/*never_exits=*/true, ProcessHandlerFn()});
}
void StubProcessFactory::SetProcessExitsImmediately(std::string command_part,
uint32_t exit_code) {
process_info_.push_back({std::move(command_part), /*std_out=*/"",
/*std_err=*/"", exit_code, /*has_exited=*/true,
/*never_exits=*/false, ProcessHandlerFn()});
}
void StubProcessFactory::SetProcessHandler(std::string command_part,
ProcessHandlerFn handler) {
process_info_.push_back({std::move(command_part), /*std_out=*/"",
/*std_err=*/"",
/*exit_code_=*/0, /*has_exited=*/false,
/*never_exits=*/false, handler});
}
} // namespace cdc_ft

78
common/stub_process.h Normal file
View File

@@ -0,0 +1,78 @@
/*
* Copyright 2022 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef COMMON_STUB_PROCESS_H_
#define COMMON_STUB_PROCESS_H_
#include <vector>
#include "absl/status/status.h"
#include "common/process.h"
namespace cdc_ft {
// A stub process returns prefabricated stdout, stderr and exit codes to
// predetermined commands.
class StubProcessFactory : public ProcessFactory {
public:
using ProcessHandlerFn = std::function<void(
std::string* std_out, std::string* std_err, uint32_t* exit_code)>;
virtual StubProcessFactory::~StubProcessFactory();
// ProcessFactory:
std::unique_ptr<Process> Create(const ProcessStartInfo& start_info) override;
// Sets up a stub process that is returned by a call to Create() if the
// command contains |command_part|. The stub process exits on RunUntil(),
// writes the given |std_out| and |std_err| to the output stream and exits
// with |exit_code|.
void SetProcessOutput(std::string command_part, std::string std_out,
std::string std_err, uint32_t exit_code);
// Sets up a stub process that is returned by a call to Create() if the
// command contains |command_part|. The process never exits. Useful for
// testing timeout conditions.
void SetProcessNeverExits(std::string command_part);
// Sets up a stub process that is returned by a call to Create() if the
// command contains |command_part|. The stub process is immediately in an
// exited state with code |exit_code| and does not write stdout or stderr.
void SetProcessExitsImmediately(std::string command_part, uint32_t exit_code);
// Sets up a stub process that is returned by a call to Create() if the
// command contains |command_part|. The stub process calls |handler| on
// RunUntil(), writes returned |std_out| and |std_err| to the output stream
// and exits with |exit_code|.
void SetProcessHandler(std::string command_part, ProcessHandlerFn handler);
private:
struct ProcessInfo {
std::string command_part;
std::string std_out;
std::string std_err;
uint32_t exit_code;
bool has_exited;
bool never_exits;
ProcessHandlerFn handler;
};
std::vector<ProcessInfo> process_info_;
};
} // namespace cdc_ft
#endif // COMMON_STUB_PROCESS_H_

53
common/test_main.cc Normal file
View File

@@ -0,0 +1,53 @@
// Copyright 2022 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "common/path.h"
#include "gtest/gtest.h"
#include "tools/cpp/runfiles/runfiles.h"
using bazel::tools::cpp::runfiles::Runfiles;
namespace cdc_ft {
namespace {
Runfiles* runfiles_ptr;
}
std::string GetTestDataDir(const char* test_data_dir) {
std::string test_path =
::testing::UnitTest::GetInstance()->current_test_info()->file();
std::string test_dir = path::DirName(test_path);
std::string root_runfile = path::ToUnix(
path::Join("cdc_file_transfer", test_dir, "testdata", "root.txt"));
assert(runfiles_ptr != nullptr);
std::string root_path = runfiles_ptr->Rlocation(root_runfile);
path::FixPathSeparators(&root_path);
EXPECT_TRUE(path::Exists(root_path))
<< path::Join(test_dir, "testdata") << " directory has no root.txt file";
return path::Join(path::DirName(root_path), test_data_dir);
}
} // namespace cdc_ft
int main(int argc, char** argv) {
std::string error;
std::unique_ptr<Runfiles> runfiles(Runfiles::Create(argv[0], &error));
if (runfiles == nullptr) {
std::cerr << "Failed to locate runtime files: " << error << std::endl;
return 1;
}
cdc_ft::runfiles_ptr = runfiles.get();
::testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}

32
common/test_main.h Normal file
View File

@@ -0,0 +1,32 @@
/*
* Copyright 2022 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef COMMON_TEST_MAIN_H_
#define COMMON_TEST_MAIN_H_
#include <string>
namespace cdc_ft {
// Retreives the full path of the testdata/|test_data_dir| directory from
// runfiles. The testdata directory must have an empty file named "root.txt".
// |test_data_dir| usually matches the filename of the test file without the
// trailing "_test.cc", e.g. "path" for "path_test.cc".
std::string GetTestDataDir(const char* test_data_dir);
} // namespace cdc_ft
#endif // COMMON_TEST_MAIN_H_

0
common/testdata/dir_iter/a/aa/aaa1.txt vendored Normal file
View File

0
common/testdata/dir_iter/a/aa/aaa2.txt vendored Normal file
View File

0
common/testdata/dir_iter/a/aa1.txt vendored Normal file
View File

0
common/testdata/dir_iter/a/aa2.txt vendored Normal file
View File

0
common/testdata/dir_iter/a/ab/aab1.txt vendored Normal file
View File

0
common/testdata/dir_iter/a/ab/aab2.txt vendored Normal file
View File

0
common/testdata/dir_iter/b/ba/bba1.txt vendored Normal file
View File

0
common/testdata/dir_iter/b/ba/bba2.txt vendored Normal file
View File

0
common/testdata/dir_iter/b/bb/bbb1.txt vendored Normal file
View File

0
common/testdata/dir_iter/b/bb/bbb2.txt vendored Normal file
View File

0
common/testdata/dir_iter/c/c1.txt vendored Normal file
View File

0
common/testdata/dir_iter/c/c2.txt vendored Normal file
View File

0
common/testdata/dir_iter/d/d1.txt vendored Normal file
View File

0
common/testdata/dir_iter/d/d2.txt vendored Normal file
View File

0
common/testdata/dir_iter/root.txt vendored Normal file
View File

1
common/testdata/gamelet_component/other/cdc_rsync_server vendored Normal file
View File

@@ -0,0 +1 @@
changed fake cdc_rsync_server

1
common/testdata/gamelet_component/valid/cdc_rsync_server vendored Normal file
View File

@@ -0,0 +1 @@
fake cdc_rsync_server

0
common/testdata/root.txt vendored Normal file
View File

44
common/testing_clock.cc Normal file
View File

@@ -0,0 +1,44 @@
// Copyright 2022 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "common/testing_clock.h"
#include "common/stopwatch.h"
namespace cdc_ft {
TestingSteadyClock::Timestamp TestingSteadyClock::Now() const {
const Timestamp prev_now = now_;
now_ += std::chrono::milliseconds(auto_advance_ms_);
return prev_now;
}
void TestingSteadyClock::Advance(int milliseconds) {
now_ += std::chrono::milliseconds(milliseconds);
}
void TestingSteadyClock::AutoAdvance(int milliseconds) {
auto_advance_ms_ = milliseconds;
}
TestingSystemClock::TestingSystemClock()
: now_(std::chrono::system_clock::now()) {}
TestingSystemClock::Timestamp TestingSystemClock::Now() const { return now_; }
void TestingSystemClock::Advance(int milliseconds) {
now_ += std::chrono::milliseconds(milliseconds);
}
} // namespace cdc_ft

56
common/testing_clock.h Normal file
View File

@@ -0,0 +1,56 @@
/*
* Copyright 2022 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef COMMON_TESTING_CLOCK_H_
#define COMMON_TESTING_CLOCK_H_
#include "common/clock.h"
namespace cdc_ft {
class TestingSteadyClock : public SteadyClock {
public:
// SteadyClock:
Timestamp Now() const override;
// Explicitly advances the timestamp by the given number of |milliseconds|.
void Advance(int milliseconds);
// Sets a time that is added to the current time after every Now() call.
void AutoAdvance(int milliseconds);
private:
mutable Timestamp now_;
int auto_advance_ms_ = 0;
};
class TestingSystemClock : public SystemClock {
public:
TestingSystemClock();
// SystemClock:
Timestamp Now() const override;
// Explicitly advances the timestamp by the given number of |milliseconds|.
void Advance(int milliseconds);
private:
Timestamp now_;
};
} // namespace cdc_ft
#endif // COMMON_TESTING_CLOCK_H_

52
common/thread_safe_map.h Normal file
View File

@@ -0,0 +1,52 @@
/*
* Copyright 2022 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef COMMON_THREAD_SAFE_MAP_H_
#define COMMON_THREAD_SAFE_MAP_H_
#include <unordered_map>
#include "absl/synchronization/mutex.h"
namespace cdc_ft {
// A map with keys of type |key_type| and values of type |mapped_type|.
template <class key_type, class mapped_type>
class ThreadSafeMap {
public:
// Returns a value by the |key|. If the |key| is not present,
// returns a default value of the `mapped_type`.
mapped_type Get(const key_type& key) const ABSL_LOCKS_EXCLUDED(mutex_) {
absl::ReaderMutexLock lock(&mutex_);
auto it = kv_.find(key);
return it == kv_.end() ? mapped_type() : it->second;
}
// Sets |value| to |key|.
void Set(const key_type& key, const mapped_type& value)
ABSL_LOCKS_EXCLUDED(mutex_) {
absl::WriterMutexLock lock(&mutex_);
kv_[key] = value;
}
private:
mutable absl::Mutex mutex_;
std::unordered_map<key_type, mapped_type> kv_ ABSL_GUARDED_BY(mutex_);
};
} // namespace cdc_ft
#endif // COMMON_THREAD_SAFE_MAP_H_

59
common/thread_safe_map_test.cc Normal file
View File

@@ -0,0 +1,59 @@
// Copyright 2022 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "common/thread_safe_map.h"
#include "gtest/gtest.h"
namespace cdc_ft {
namespace {
class ThreadSafeMapTest : public ::testing::Test {};
TEST_F(ThreadSafeMapTest, SetGetExising) {
ThreadSafeMap<std::string, std::string> map_str_str;
std::string key_str = "key 1";
std::string val_str = "val 1";
map_str_str.Set(key_str, val_str);
EXPECT_EQ(map_str_str.Get(key_str), val_str);
ThreadSafeMap<int, bool> map_int_bool;
int key_int = 345;
bool val_bool = true;
map_int_bool.Set(key_int, val_bool);
EXPECT_EQ(map_int_bool.Get(key_int), val_bool);
ThreadSafeMap<int, double> map_int_double;
key_int = 124;
double val_double = 987.23445432;
map_int_double.Set(key_int, val_double);
EXPECT_EQ(map_int_double.Get(key_int), val_double);
}
TEST_F(ThreadSafeMapTest, SetGetMissing) {
ThreadSafeMap<std::string, std::string> map_str_str;
map_str_str.Set("key 1", "val 1");
EXPECT_EQ(map_str_str.Get("key 2"), "");
ThreadSafeMap<int, bool> map_int_bool;
map_int_bool.Set(1287, true);
EXPECT_EQ(map_int_bool.Get(-98), false);
ThreadSafeMap<int, double> map_int_double;
map_int_double.Set(-82534, 876.121232);
EXPECT_EQ(map_int_double.Get(12), 0);
}
} // namespace
} // namespace cdc_ft

122
common/threadpool.cc Normal file
View File

@@ -0,0 +1,122 @@
// Copyright 2022 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "common/threadpool.h"
namespace cdc_ft {
Threadpool::Threadpool(size_t num_threads) : shutdown_(false) {
workers_.reserve(num_threads);
for (size_t n = 0; n < num_threads; ++n) {
workers_.emplace_back([this]() { ThreadWorkerMain(); });
}
}
Threadpool::~Threadpool() { Shutdown(); }
void Threadpool::Wait() {
absl::MutexLock lock(&task_queue_mutex_);
auto cond = [this]() ABSL_EXCLUSIVE_LOCKS_REQUIRED(task_queue_mutex_) {
return outstanding_task_count_ == 0;
};
task_queue_mutex_.Await(absl::Condition(&cond));
}
void Threadpool::Shutdown() {
{
// Signal shutdown.
absl::MutexLock lock(&task_queue_mutex_);
if (shutdown_) return;
shutdown_ = true;
}
// Join thread. This makes sure that the last task finishes.
for (auto& worker : workers_) {
if (worker.joinable()) worker.join();
}
}
void Threadpool::QueueTask(std::unique_ptr<Task> task) {
absl::MutexLock lock(&task_queue_mutex_);
++outstanding_task_count_;
task_queue_.push(std::move(task));
}
std::unique_ptr<Task> Threadpool::TryGetCompletedTask() {
absl::MutexLock lock(&completed_tasks_mutex_);
if (completed_tasks_.empty()) {
return std::unique_ptr<Task>();
}
std::unique_ptr<Task> task = std::move(completed_tasks_.front());
completed_tasks_.pop();
return task;
}
std::unique_ptr<Task> Threadpool::GetCompletedTask() {
absl::MutexLock lock(&completed_tasks_mutex_);
auto cond = [this]() ABSL_EXCLUSIVE_LOCKS_REQUIRED(completed_tasks_mutex_) {
return !completed_tasks_.empty();
};
completed_tasks_mutex_.Await(absl::Condition(&cond));
std::unique_ptr<Task> task = std::move(completed_tasks_.front());
completed_tasks_.pop();
return task;
}
void Threadpool::ThreadWorkerMain() {
bool task_finished = false;
for (;;) {
std::unique_ptr<Task> task;
{
absl::MutexLock lock(&task_queue_mutex_);
// Decrease task count here, so we don't have to lock again at the end of
// the loop.
if (task_finished) {
assert(outstanding_task_count_ > 0);
--outstanding_task_count_;
}
// Wait for task to be available (or shutdown).
auto cond = [this]() ABSL_EXCLUSIVE_LOCKS_REQUIRED(task_queue_mutex_) {
return shutdown_ || !task_queue_.empty();
};
task_queue_mutex_.Await(absl::Condition(&cond));
if (shutdown_) break;
// Grab task from queue.
task = std::move(task_queue_.front());
task_queue_.pop();
}
// Run task, but make it cancellable.
task->ThreadRun([this]() ABSL_EXCLUSIVE_LOCKS_REQUIRED(
task_queue_mutex_) -> bool { return shutdown_; });
{
absl::MutexLock lock(&task_queue_mutex_);
if (shutdown_) break;
}
// Push task to completed queue.
absl::MutexLock lock(&completed_tasks_mutex_);
completed_tasks_.push(std::move(task));
task_finished = true;
}
}
} // namespace cdc_ft

103
common/threadpool.h Normal file
View File

@@ -0,0 +1,103 @@
/*
* Copyright 2022 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef COMMON_THREADPOOL_H_
#define COMMON_THREADPOOL_H_
#include <atomic>
#include <condition_variable>
#include <functional>
#include <memory>
#include <queue>
#include <thread>
#include <vector>
#include "absl/synchronization/mutex.h"
namespace cdc_ft {
class Task {
public:
using IsCancelledPredicate = std::function<bool()>;
virtual ~Task() = default;
// Method that's doing all the work.
// Called on a background thread.
virtual void ThreadRun(IsCancelledPredicate is_cancelled) = 0;
};
// Manages a pool of worker threads and schedules tasks to run on the threads.
class Threadpool {
public:
// Creates a new thread pool with |num_threads| worker threads.
explicit Threadpool(size_t num_threads);
~Threadpool();
// Waits for all queued tasks to finish.
void Wait() ABSL_LOCKS_EXCLUDED(task_queue_mutex_);
// Stops the worker threads. Cancels currently active tasks.
void Shutdown() ABSL_LOCKS_EXCLUDED(task_queue_mutex_);
// Queues a task for execution in a worker thread.
void QueueTask(std::unique_ptr<Task> task)
ABSL_LOCKS_EXCLUDED(task_queue_mutex_);
// If available, returns the next completed task.
// For a single worker thread (|num_threads| == 1), tasks are completed in
// FIFO order. This is no longer the case for multiple threads
// (|num_threads| > 1). Tasks that got queued later might complete first.
std::unique_ptr<Task> TryGetCompletedTask()
ABSL_LOCKS_EXCLUDED(completed_tasks_mutex_);
// Returns the next completed task, possibly blocking until it is available.
// For a single worker thread (|num_threads| == 1), tasks are completed in
// FIFO order. This is no longer the case for multiple threads
// (|num_threads| > 1). Tasks that got queued later might complete first.
std::unique_ptr<Task> GetCompletedTask()
ABSL_LOCKS_EXCLUDED(completed_tasks_mutex_);
// Returns the total number of worker threads in the pool.
size_t NumThreads() const { return workers_.size(); }
// Returns the number of tasks that are either queued or in progress.
size_t NumQueuedTasks() const ABSL_LOCKS_EXCLUDED(task_queue_mutex_) {
absl::ReaderMutexLock lock(&task_queue_mutex_);
return outstanding_task_count_;
}
private:
// Background thread worker method. Picks tasks and runs them.
void ThreadWorkerMain()
ABSL_LOCKS_EXCLUDED(task_queue_mutex_, completed_tasks_mutex_);
mutable absl::Mutex task_queue_mutex_;
std::queue<std::unique_ptr<Task>> task_queue_
ABSL_GUARDED_BY(task_queue_mutex_);
size_t outstanding_task_count_ ABSL_GUARDED_BY(task_queue_mutex_) = 0;
std::atomic_bool shutdown_ ABSL_GUARDED_BY(task_queue_mutex_);
absl::Mutex completed_tasks_mutex_;
std::queue<std::unique_ptr<Task>> completed_tasks_
ABSL_GUARDED_BY(completed_tasks_mutex_);
std::vector<std::thread> workers_;
};
} // namespace cdc_ft
#endif // COMMON_THREADPOOL_H_

155
common/threadpool_test.cc Normal file
View File

@@ -0,0 +1,155 @@
// Copyright 2022 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "common/threadpool.h"
#include <atomic>
#include <functional>
#include <unordered_set>
#include "common/semaphore.h"
#include "common/util.h"
#include "gtest/gtest.h"
namespace cdc_ft {
namespace {
// Wrapper class to make it possible to pass lambdas as task functions.
class TestTask : public Task {
public:
using TaskFunc = std::function<void(IsCancelledPredicate is_cancelled)>;
explicit TestTask(TaskFunc task_func) : task_func_(task_func) {}
virtual void ThreadRun(IsCancelledPredicate is_cancelled) {
task_func_(is_cancelled);
}
private:
TaskFunc task_func_;
};
class ThreadpoolTest : public ::testing::Test {};
TEST_F(ThreadpoolTest, WaitShutdownWorkWithoutTasks) {
Threadpool pool(3);
pool.Wait();
pool.Shutdown();
}
TEST_F(ThreadpoolTest, SingleThreadedRunsToCompletion) {
std::atomic_bool task_finished(false);
auto task_func = [&task_finished](Task::IsCancelledPredicate) {
task_finished = true;
};
Threadpool pool(1);
std::unique_ptr<Task> task_ptr = std::make_unique<TestTask>(task_func);
Task* task = task_ptr.get();
pool.QueueTask(std::move(task_ptr));
pool.Wait();
EXPECT_TRUE(task_finished);
std::unique_ptr<Task> completed_task = pool.TryGetCompletedTask();
EXPECT_EQ(completed_task.get(), task);
}
TEST_F(ThreadpoolTest, MultiThreadedRunsToCompletion) {
const int num_tasks = 19;
const int num_threads = 7;
std::atomic_int num_completed(0);
Threadpool pool(num_threads);
std::unordered_set<Task*> tasks;
for (int n = 0; n < num_tasks; ++n) {
auto task_func = [&num_completed](Task::IsCancelledPredicate) {
++num_completed;
};
auto task = std::make_unique<TestTask>(task_func);
tasks.insert(task.get());
pool.QueueTask(std::move(task));
}
pool.Wait();
EXPECT_EQ(num_completed, num_tasks);
for (int n = 0; n < num_tasks; ++n) {
std::unique_ptr<Task> completed_task = pool.TryGetCompletedTask();
EXPECT_TRUE(completed_task);
tasks.erase(completed_task.get());
}
EXPECT_FALSE(pool.TryGetCompletedTask());
EXPECT_TRUE(tasks.empty());
}
TEST_F(ThreadpoolTest, TaskIsCancelledOnShutdown) {
Semaphore task_started(0);
std::atomic_bool task_finished(false);
auto task_func = [&task_started,
&task_finished](Task::IsCancelledPredicate is_cancelled) {
task_started.Signal();
while (!is_cancelled()) {
Util::Sleep(0);
}
task_finished = true;
};
Threadpool pool(1);
pool.QueueTask(std::make_unique<TestTask>(task_func));
task_started.Wait();
pool.Shutdown();
EXPECT_TRUE(task_finished);
// The cancelled task should be discarded.
std::unique_ptr<Task> completed_task = pool.TryGetCompletedTask();
EXPECT_FALSE(completed_task.get());
}
TEST_F(ThreadpoolTest, SingleThreadedCompletesInFifoOrder) {
const int num_tasks = 10;
std::vector<int> completed_order;
Threadpool pool(1);
for (int n = 0; n < num_tasks; ++n) {
auto task_func = [n, &completed_order](Task::IsCancelledPredicate) {
// Thread-safe because there's only one worker thread.
completed_order.push_back(n);
};
pool.QueueTask(std::make_unique<TestTask>(task_func));
}
pool.Wait();
std::unique_ptr<Task> completed_task = pool.TryGetCompletedTask();
ASSERT_EQ(completed_order.size(), num_tasks);
for (int n = 0; n < num_tasks; ++n) {
EXPECT_EQ(completed_order[n], n);
}
}
TEST_F(ThreadpoolTest, GetCompletedTask) {
auto task_func = [](Task::IsCancelledPredicate) { Util::Sleep(10); };
Threadpool pool(1);
std::unique_ptr<Task> task_ptr = std::make_unique<TestTask>(task_func);
Task* task = task_ptr.get();
pool.QueueTask(std::move(task_ptr));
// Note: No pool.Wait().
std::unique_ptr<Task> completed_task = pool.GetCompletedTask();
EXPECT_EQ(completed_task.get(), task);
}
} // namespace
} // namespace cdc_ft

59
common/url.cc Normal file
View File

@@ -0,0 +1,59 @@
// Copyright 2022 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "common/url.h"
#include <ctype.h>
#include <regex>
#include "absl/strings/str_format.h"
namespace cdc_ft {
absl::StatusOr<Url> Url::Parse(const std::string& url_string) {
std::string url_copy = url_string;
std::for_each(url_copy.begin(), url_copy.end(),
[](char& c) { c = tolower(c); });
std::regex re("(https?)://([^:^?/]+)(?::([0-9]+))?.*");
std::smatch match;
if (!std::regex_match(url_copy, match, re))
return absl::Status(
absl::StatusCode::kInvalidArgument,
absl::StrFormat("'%s' is not a valid url.", url_string.c_str()));
Url url;
url.protocol = match[1].str();
url.host = match[2].str();
if (match.size() == 4 && match[3].length() > 0)
url.port = match[3].str();
else if (url.protocol.back() == 's')
url.port = "443";
else
url.port = "80";
return url;
}
Url::Url() = default;
Url::Url(std::string protocol, std::string host, std::string port)
: protocol(protocol), host(host), port(port) {}
Url::~Url() = default;
bool operator==(const Url& lhs, const Url& rhs) {
return lhs.host == rhs.host && lhs.port == rhs.port &&
lhs.protocol == rhs.protocol;
}
} // namespace cdc_ft

42
common/url.h Normal file
View File

@@ -0,0 +1,42 @@
/*
* Copyright 2022 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef COMMON_URL_H_
#define COMMON_URL_H_
#include "absl/status/statusor.h"
namespace cdc_ft {
struct Url {
Url();
Url(std::string protocol, std::string host, std::string port);
~Url();
std::string protocol;
std::string host;
std::string port;
// Parses |url_string| to Url struct from a given HTTP(S) URL.
// Uses a very simple parser, e.g. does not unescape host (e.g. foo%20bar).
static absl::StatusOr<Url> Parse(const std::string& url_string);
};
bool operator==(const Url& lhs, const Url& rhs);
} // namespace cdc_ft
#endif // COMMON_URL_H_

52
common/url_test.cc Normal file
View File

@@ -0,0 +1,52 @@
// Copyright 2022 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "common/url.h"
#include "absl/status/statusor.h"
#include "common/status_test_macros.h"
#include "gtest/gtest.h"
namespace cdc_ft {
namespace {
class UrlTest : public ::testing::Test {
protected:
void AssertEqual(absl::StatusOr<Url> actual, Url expected) {
EXPECT_OK(actual);
EXPECT_EQ(actual.value(), expected);
}
};
TEST_F(UrlTest, ParseUrl) {
AssertEqual(Url::Parse("https://hOSt.t.gF.d"),
Url("https", "host.t.gf.d", "443"));
AssertEqual(Url::Parse("http://h"), Url("http", "h", "80"));
AssertEqual(Url::Parse("https://h:8080/path"), Url("https", "h", "8080"));
AssertEqual(Url::Parse("https://h?t=45"), Url("https", "h", "443"));
AssertEqual(Url::Parse("https://h:765?t=45"), Url("https", "h", "765"));
AssertEqual(
Url::Parse(
"https://my-domain.ho.HO.ho:80/path/to/smth?arg1=123&dfd=[g,6,y]"),
Url("https", "my-domain.ho.ho.ho", "80"));
EXPECT_NOT_OK(Url::Parse("httpss://domain"));
EXPECT_NOT_OK(Url::Parse("HOST.com:443/path"));
EXPECT_NOT_OK(Url::Parse("https:/domain"));
EXPECT_NOT_OK(Url::Parse("http//domain"));
EXPECT_NOT_OK(Url::Parse("htps://dmn.ua"));
EXPECT_NOT_OK(Url::Parse("non-url"));
}
} // namespace
} // namespace cdc_ft

298
common/util.cc Normal file
View File

@@ -0,0 +1,298 @@
// Copyright 2022 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "common/util.h"
#include <errno.h>
#include <cassert>
#include "absl/random/random.h"
#include "absl/strings/str_format.h"
#if PLATFORM_LINUX
#include <sys/ioctl.h> // struct winsize, TIOCGWINSZ
#include <unistd.h> // usleep, STDOUT_FILENO, isatty
#elif PLATFORM_WINDOWS
#include <io.h> // _isatty
#define WIN32_LEAN_AND_MEAN
#include <windows.h>
#endif
namespace cdc_ft {
namespace {
constexpr char kGuidFormat[] = "%08lX-%04hX-%04hX-%04hX-%04hX%08lX";
}
#if PLATFORM_WINDOWS
// static
std::string Util::WideToUtf8Str(const std::wstring& wchar_str) {
int wchar_size = static_cast<int>(wchar_str.size());
int utf8_size = WideCharToMultiByte(CP_UTF8, 0, wchar_str.c_str(), wchar_size,
nullptr, 0, nullptr, nullptr);
assert(utf8_size != 0 || wchar_size == 0);
std::string utf8_str(utf8_size, 0);
WideCharToMultiByte(
CP_UTF8, 0, wchar_str.c_str(), static_cast<int>(wchar_str.size()),
const_cast<LPSTR>(utf8_str.data()), utf8_size, nullptr, 0);
return utf8_str;
}
#endif // #if PLATFORM_WINDOWS
#if PLATFORM_WINDOWS
// static
std::wstring Util::Utf8ToWideStr(const std::string& utf8_str) {
int utf8_size = static_cast<int>(utf8_str.size());
int wchar_size =
MultiByteToWideChar(CP_UTF8, 0, utf8_str.c_str(), utf8_size, nullptr, 0);
assert(wchar_size != 0 || utf8_size == 0);
std::wstring wchar_str(wchar_size, 0);
MultiByteToWideChar(CP_UTF8, 0, utf8_str.c_str(), utf8_size,
const_cast<LPWSTR>(wchar_str.data()), wchar_size);
return wchar_str;
}
#endif // #if PLATFORM_WINDOWS
#if PLATFORM_WINDOWS
// static
std::string Util::GetWin32Error(uint32_t error) {
LPVOID lpMsgBuf;
FormatMessage(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM |
FORMAT_MESSAGE_IGNORE_INSERTS,
NULL, error, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT),
reinterpret_cast<LPTSTR>(&lpMsgBuf), 0, NULL);
std::string message = WideToUtf8Str((LPCTSTR)lpMsgBuf);
LocalFree(lpMsgBuf);
while (!message.empty() &&
(message.back() == '\r' || message.back() == '\n')) {
message.pop_back();
}
return message;
}
#endif // #if PLATFORM_WINDOWS
#if PLATFORM_WINDOWS
// static
std::string Util::GetLastWin32Error() { return GetWin32Error(GetLastError()); }
#endif // #if PLATFORM_WINDOWS
// static
std::string Util::GetStrError(int err) {
#if PLATFORM_WINDOWS
char msg[80] = {0};
strerror_s(msg, sizeof(msg) - 1, err);
return msg;
#elif PLATFORM_LINUX
return std::strerror(err);
#endif
}
// static
std::string Util::GetLastStrError() { return GetStrError(errno); }
// static
int64_t Util::GetPid() {
#if PLATFORM_WINDOWS
return ::GetCurrentProcessId();
#elif PLATFORM_LINUX
return getpid();
#endif
}
// static
int Util::GetConsoleWidth() {
static constexpr int kDefaultConsoleWidth = 80;
#if PLATFORM_WINDOWS
CONSOLE_SCREEN_BUFFER_INFO info;
if (!GetConsoleScreenBufferInfo(GetStdHandle(STD_OUTPUT_HANDLE), &info)) {
return kDefaultConsoleWidth;
}
return info.srWindow.Right - info.srWindow.Left;
#elif PLATFORM_LINUX
struct winsize size;
if (ioctl(STDOUT_FILENO, TIOCGWINSZ, &size) != 0) {
return kDefaultConsoleWidth;
}
return size.ws_col;
#endif
}
// static
bool Util::IsTTY() {
#if PLATFORM_WINDOWS
return _isatty(_fileno(stdout));
#elif PLATFORM_LINUX
return isatty(STDOUT_FILENO);
#endif
}
// static
bool Util::IsExecutable(const void* data, size_t size) {
static constexpr uint8_t kElfMagic[] = {0x7fu, 0x45u, 0x4cu, 0x46u};
static constexpr uint8_t kSheBangMagic[] = {'#', '!', '/'};
static constexpr uint8_t kMzMagic[] = {0x4du, 0x5au};
if (size < std::max(std::max(sizeof kElfMagic, sizeof kSheBangMagic),
sizeof kMzMagic)) {
return false;
}
if (memcmp(data, kElfMagic, sizeof(kElfMagic)) == 0) return true;
if (memcmp(data, kSheBangMagic, sizeof(kSheBangMagic)) == 0) return true;
if (memcmp(data, kMzMagic, sizeof(kMzMagic)) == 0) return true;
return false;
}
// static
void Util::WaitForDebugger() {
#if PLATFORM_LINUX
static int s = 0;
while (s == 0) {
Sleep(1);
}
#elif PLATFORM_WINDOWS
__debugbreak();
#endif
}
// static
void Util::Sleep(uint32_t duration_ms) {
#if PLATFORM_LINUX
usleep(duration_ms * 1000);
#elif PLATFORM_WINDOWS
::Sleep(duration_ms);
#endif
}
// static
int Util::Utf8CodePointLen(const char* str) {
// UTF-8 code points are encoded as follows:
// 1-byte: 0xxxxxxx
// 2-byte: 110xxxxx 10xxxxxx
// 3-byte: 1110xxxx 10xxxxxx 10xxxxxx
// 4-byte: 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
#define Is10(index) ((ustr[index] & 0xC0) == 0x80)
// Note that this is guaranteed to never access memory beyond \0.
const uint8_t* ustr = reinterpret_cast<const uint8_t*>(str);
if (ustr[0] == 0) return 0;
if ((ustr[0] & 0x80) == 0x00) return 1;
if ((ustr[0] & 0xE0) == 0xC0) return Is10(1) ? 2 : 0;
if ((ustr[0] & 0xF0) == 0xE0) return Is10(1) && Is10(2) ? 3 : 0;
if ((ustr[0] & 0xF8) == 0xF0) return Is10(1) && Is10(2) && Is10(3) ? 4 : 0;
return 0;
#undef Is10
}
std::string Util::GenerateUniqueId() {
absl::BitGen gen;
return absl::StrFormat(
kGuidFormat, absl::Uniform<uint32_t>(gen), absl::Uniform<uint16_t>(gen),
absl::Uniform<uint16_t>(gen), absl::Uniform<uint16_t>(gen),
absl::Uniform<uint16_t>(gen), absl::Uniform<uint32_t>(gen));
}
std::vector<absl::string_view> SplitString(absl::string_view s,
const char delim, bool keep_empty) {
if (s.empty()) return std::vector<absl::string_view>();
// Count no. of parts so that we can pre-allocate the target vector.
int count = 1;
size_t pos = 0;
size_t last_pos = std::string::npos;
for (pos = s.find(delim, pos); pos != std::string::npos;
pos = s.find(delim, pos + 1)) {
if (keep_empty || pos - 1 != last_pos) ++count;
last_pos = pos;
}
// Collect all parts
std::vector<absl::string_view> parts;
parts.reserve(count);
size_t first = 0, last = 0;
for (last = s.find(delim, first); last != std::string::npos;
last = s.find(delim, first)) {
if (keep_empty || first < last) {
parts.push_back(absl::string_view(s.data() + first, last - first));
}
first = last + 1;
}
// Append the last part, if needed
if (keep_empty || first < s.size()) {
parts.push_back(absl::string_view(s.data() + first, s.size() - first));
}
return parts;
}
std::vector<absl::string_view> SplitString(const std::string& s,
const char delim, bool keep_empty) {
absl::string_view sv(s.c_str(), s.size());
return SplitString(sv, delim, keep_empty);
}
std::string HumanBytes(double size, int precision) {
const double threshold = 2048;
if (size < 1024)
return absl::StrFormat("%d bytes", static_cast<size_t>(size));
size /= 1024.0;
std::string units = "KB";
if (size > threshold) {
size /= 1024.0;
units = "MB";
}
if (size > threshold) {
size /= 1024.0;
units = "GB";
}
if (size > threshold) {
size /= 1024.0;
units = "TB";
}
if (size > threshold) {
size /= 1024.0;
units = "PB";
}
return absl::StrFormat("%.*f %s", precision, size, units);
}
std::string HumanDuration(const absl::Duration& d) {
auto sec = absl::ToInt64Seconds(d);
return absl::StrFormat("%02d:%02d", sec / 60, std::abs(sec) % 60);
}
std::string JoinStrings(const std::vector<absl::string_view>& parts,
size_t first, size_t last, const char delim) {
std::string ret;
for (size_t i = first; i < last && i < parts.size(); ++i) {
if (i != first) ret.push_back(delim);
ret.append(parts[i]);
}
return ret;
}
std::string JoinStrings(const std::vector<absl::string_view>& parts,
const char delim) {
if (parts.empty()) return std::string();
return JoinStrings(parts, 0, parts.size(), delim);
}
} // namespace cdc_ft

137
common/util.h Normal file
View File

@@ -0,0 +1,137 @@
/*
* Copyright 2022 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef COMMON_UTIL_H_
#define COMMON_UTIL_H_
#include <string>
#include <vector>
#include "absl/strings/string_view.h"
#include "absl/time/time.h"
#include "common/platform.h"
namespace cdc_ft {
// Assorted utilities
class Util {
public:
#if PLATFORM_WINDOWS
// Converts a wide character string to a UTF8 string.
// Illegal sequences are replaced by U+FFFD.
static std::string WideToUtf8Str(const std::wstring& wchar_str);
// Converts a UTF8 string to a wide character string.
// Illegal sequences are replaced by U+FFFD.
static std::wstring Utf8ToWideStr(const std::string& utf8_str);
// Returns the string that corresponds to the Windows error id |error|.
static std::string GetWin32Error(uint32_t error);
// Returns the string that corresponds to the Windows error id GetLastError().
static std::string GetLastWin32Error();
#endif
// Returns the C runtime error for |err|.
static std::string GetStrError(int err);
// Returns the last C runtime error.
static std::string GetLastStrError();
// Returns the id of the current process.
static int64_t GetPid();
// Returns the width or kDefaultConsoleWidth if not running in console mode.
static int GetConsoleWidth();
// Returns true if stdout is associated with a terminal (alias TTY).
static bool IsTTY();
// Determines whether the given |data| matches PE/elf/shebang magic headers.
// The data should be the beginning of a file and at least 4 bytes long.
// The detection might yield false positives, but no false negatives.
static bool IsExecutable(const void* data, size_t size);
// On Windows, waits for a debugger to be attached and starts debugging.
// On Linux, this just runs an infinite loop. To get into the debugger, set
// breakpoint in the loop and move the instruction pointer (yellow arrow in
// Visual Studio) out of the loop.
static void WaitForDebugger();
// Sleeps for |duration_ms| milliseconds.
static void Sleep(uint32_t duration_ms);
// Returns the number of bytes of the first UTF8 code point in str.
// Returns 0 if the code point is not valid.
static int Utf8CodePointLen(const char* str);
// Generates a unique identifier. The identifier is a sequence of base-16
// digits in the following format: '01234567-89AB-CDEF-0123-456789AB'.
static std::string GenerateUniqueId();
};
// Splits the given string |s| at the given delimiter |delim| and returns the
// parts as a vector of string_views. Since a string_view does not copy the
// data, the caller is responsible for keeping the string memory allocated as
// long as the referenced parts need to be accessed.
//
// If |keep_empty| is false, empty parts will not be included in the result,
// otherwise empty string_views might be inserted.
//
// Returns an empty list if the given string is empty. If the delimiter is not
// found in |s|, a list with a single element containing |s| is returned.
std::vector<absl::string_view> SplitString(absl::string_view s,
const char delim,
bool keep_empty = true);
std::vector<absl::string_view> SplitString(const std::string& s,
const char delim,
bool keep_empty = true);
// Combines the given string |parts| to one string, separated by |delim|. Does
// not check if the previous part already ended with |delim|.
std::string JoinStrings(const std::vector<absl::string_view>& parts,
const char delim);
// Combines the given string |parts| in the range [first, last) to one string,
// all separated by |delim|. Does not check if the previous part already ended
// with |delim|. Ignores indices that are out of bounds.
std::string JoinStrings(const std::vector<absl::string_view>& parts,
size_t first, size_t last, const char delim);
// Prints a human-readable representation of the given |size| and decimal
// |precision|, such as "4 KB" or "2.34 MB".
std::string HumanBytes(double size, int precision = 0);
// Prints a human-readable representation of a duration as minutes and seconds
// in the format "mm:ss" (with leading zero for > 10 minutes).
std::string HumanDuration(const absl::Duration& d);
// FinalSetter sets a given |receiver| variable to a given |value| once it gets
// out of scope.
template <typename T>
class FinalSetter {
public:
FinalSetter(T* receiver, T value) : receiver_(receiver), value_(value) {}
~FinalSetter() { *receiver_ = std::move(value_); }
private:
T* receiver_;
T value_;
};
} // namespace cdc_ft
#endif // COMMON_UTIL_H_

288
common/util_test.cc Normal file
View File

@@ -0,0 +1,288 @@
// Copyright 2022 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "common/util.h"
#include <limits>
#include "gtest/gtest.h"
namespace cdc_ft {
namespace {
using StringList = std::vector<absl::string_view>;
#if PLATFORM_WINDOWS
constexpr wchar_t kWcharString[] = L"Hey Google, where's the next ⛽?";
constexpr char kUtf8String[] = u8"Hey Google, where's the next ⛽?";
// Invalid character (\uFFFD) + terminator in oth cases.
constexpr wchar_t kInvalidWcharString[] = {65533, 0};
constexpr char kInvalidUtf8String[] = {-17, -65, -67, 0};
#endif
#if PLATFORM_WINDOWS
TEST(UtilTest, WideToUtf8Str_Valid) {
std::string utf8_string = Util::WideToUtf8Str(kWcharString);
EXPECT_EQ(utf8_string, kUtf8String);
}
#endif
#if PLATFORM_WINDOWS
TEST(UtilTest, WideToUtf8Str_Empty) {
std::string utf8_string = Util::WideToUtf8Str(L"");
EXPECT_TRUE(utf8_string.empty());
}
#endif
#if PLATFORM_WINDOWS
TEST(UtilTest, WideToUtf8Str_Invalid) {
std::wstring wchar_string = Util::Utf8ToWideStr(kInvalidUtf8String);
EXPECT_EQ(wchar_string, kInvalidWcharString);
}
#endif
#if PLATFORM_WINDOWS
TEST(UtilTest, Utf8ToWideStr_Valid) {
std::wstring wchar_string = Util::Utf8ToWideStr(kUtf8String);
EXPECT_EQ(wchar_string, kWcharString);
}
#endif
#if PLATFORM_WINDOWS
TEST(UtilTest, Utf8ToWideStr_Empty) {
std::wstring wchar_string = Util::Utf8ToWideStr("");
EXPECT_TRUE(wchar_string.empty());
}
#endif
#if PLATFORM_WINDOWS
TEST(UtilTest, Utf8ToWideStr_Invalid) {
std::string utf8_string = Util::WideToUtf8Str(kInvalidWcharString);
EXPECT_EQ(utf8_string, kInvalidUtf8String);
}
#endif
#if PLATFORM_WINDOWS
TEST(UtilTest, GetErrorString) {
EXPECT_EQ(Util::GetWin32Error(0), "The operation completed successfully.");
EXPECT_EQ(Util::GetWin32Error(1), "Incorrect function.");
}
#endif
#if PLATFORM_WINDOWS
TEST(UtilTest, GetLastErrorString) {
// Just get some coverage. We don't know the last error.
EXPECT_NE(Util::GetLastWin32Error(), "");
}
#endif
TEST(UtilTest, GetStrError) {
#if PLATFORM_WINDOWS
EXPECT_EQ(Util::GetStrError(0), "No error");
#else
EXPECT_EQ(Util::GetStrError(0), "Success");
#endif
EXPECT_EQ(Util::GetStrError(1), "Operation not permitted");
}
TEST(UtilTest, GetLastStrError) {
// Just get some coverage. We don't know the last error.
EXPECT_NE(Util::GetLastStrError(), "");
}
TEST(UtilTest, GetPid) {
// Just get some coverage. Processes are guaranteed not to return int64 min.
// On Linux, it is an int32, on Windows an uint32.
EXPECT_NE(Util::GetPid(), std::numeric_limits<int64_t>::min());
}
TEST(UtilTest, GetConsoleWidth) {
// Could be run from a console window, we don't know.
EXPECT_GT(Util::GetConsoleWidth(), 0);
}
TEST(UtilTest, IsTTY) {
// Just exercise code, we don't know how the test is run.
Util::IsTTY();
}
TEST(UtilTest, IsExecutable) {
static constexpr uint8_t kWindowsExe[] = {0x4d, 0x5a, 0x90, 0x00, 0x03, 0x00,
0x00, 0x00, 0x04, 0x00, 0x00, 0x00,
0xff, 0xff, 0x00, 0x00};
static constexpr uint8_t kLinuxElf[] = {0x7f, 0x45, 0x4c, 0x46, 0x02, 0x01,
0x01, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00};
static constexpr uint8_t kBashScript[] = {0x23, 0x21, 0x2f, 0x62, 0x69, 0x6e,
0x2f, 0x73, 0x68, 0x0d, 0x0a, 0x0d,
0x0a, 0x65, 0x63, 0x68};
static constexpr uint8_t kSecretFile[] = {0x44, 0x45, 0x46, 0x47, 0x41, 0x41,
0x48, 0x48, 0x48, 0x48, 0x41, 0x48,
0x48, 0x48, 0x48, 0x41};
EXPECT_TRUE(Util::IsExecutable(kWindowsExe, sizeof(kWindowsExe)));
EXPECT_TRUE(Util::IsExecutable(kLinuxElf, sizeof(kLinuxElf)));
EXPECT_TRUE(Util::IsExecutable(kBashScript, sizeof(kBashScript)));
EXPECT_FALSE(Util::IsExecutable(kSecretFile, sizeof(kSecretFile)));
}
TEST(UtilTest, Sleep) {
// Just get some coverage.
Util::Sleep(0);
}
TEST(UtilTest, Utf8CodePointLen) {
EXPECT_EQ(Util::Utf8CodePointLen(u8""), 0);
EXPECT_EQ(Util::Utf8CodePointLen(u8"a"), 1);
EXPECT_EQ(Util::Utf8CodePointLen(u8"ab"), 1);
EXPECT_EQ(Util::Utf8CodePointLen(u8"\u0200"), 2);
EXPECT_EQ(Util::Utf8CodePointLen(u8"\u5000"), 3);
EXPECT_EQ(Util::Utf8CodePointLen(u8"\U00010000"), 4);
EXPECT_EQ(Util::Utf8CodePointLen(u8"\u0200 only the first char counts"), 2);
// Corrupt some UTF8 character.
char broken[] = u8"\U00010000";
broken[strlen(broken) - 1] = 'd';
EXPECT_EQ(Util::Utf8CodePointLen(broken), 0);
}
TEST(UtilTest, SplitStringSkipEmpty) {
EXPECT_EQ(SplitString(std::string(""), '/', false), StringList());
EXPECT_EQ(SplitString(std::string("a"), '/', false), StringList({"a"}));
EXPECT_EQ(SplitString(std::string("a/b"), '/', false),
StringList({"a", "b"}));
EXPECT_EQ(SplitString(std::string("a//b"), '/', false),
StringList({"a", "b"}));
EXPECT_EQ(SplitString(std::string("/a/b"), '/', false),
StringList({"a", "b"}));
EXPECT_EQ(SplitString(std::string("a/b/"), '/', false),
StringList({"a", "b"}));
EXPECT_EQ(SplitString(std::string("/a/b/"), '/', false),
StringList({"a", "b"}));
EXPECT_EQ(SplitString(std::string("//a//b//"), '/', false),
StringList({"a", "b"}));
EXPECT_EQ(SplitString(std::string("aa/bb/cc"), '/', false),
StringList({"aa", "bb", "cc"}));
}
TEST(UtilTest, SplitStringKeepEmpty) {
EXPECT_EQ(SplitString(std::string(""), ',', true), StringList());
EXPECT_EQ(SplitString(std::string("a"), ',', true), StringList({"a"}));
EXPECT_EQ(SplitString(std::string("a,b"), ',', true), StringList({"a", "b"}));
EXPECT_EQ(SplitString(std::string("a,b,"), ',', true),
StringList({"a", "b", ""}));
EXPECT_EQ(SplitString(std::string("a,,c"), ',', true),
StringList({"a", "", "c"}));
EXPECT_EQ(SplitString(std::string(",b,c"), ',', true),
StringList({"", "b", "c"}));
EXPECT_EQ(SplitString(std::string(",,"), ',', true),
StringList({"", "", ""}));
EXPECT_EQ(SplitString(std::string("aa,bb,"), ',', true),
StringList({"aa", "bb", ""}));
EXPECT_EQ(SplitString(std::string("aa,,cc"), ',', true),
StringList({"aa", "", "cc"}));
EXPECT_EQ(SplitString(std::string(",bb,cc"), ',', true),
StringList({"", "bb", "cc"}));
}
TEST(UtilTest, JoinStrings) {
EXPECT_EQ(JoinStrings(StringList(), ','), std::string());
EXPECT_EQ(JoinStrings(StringList({"a"}), ','), "a");
EXPECT_EQ(JoinStrings(StringList({"a", "b"}), ','), "a,b");
EXPECT_EQ(JoinStrings(StringList({"a", "b", "c"}), ','), "a,b,c");
EXPECT_EQ(JoinStrings(StringList({"a", "b", "c"}), ' '), "a b c");
EXPECT_EQ(JoinStrings(StringList({",", ","}), ','), ",,,");
EXPECT_EQ(JoinStrings(StringList({"", ""}), ','), ",");
}
TEST(UtilTest, JoinStringsPartial) {
EXPECT_EQ(JoinStrings(StringList(), 1, 2, ','), std::string());
EXPECT_EQ(JoinStrings(StringList(), 2, 1, ','), std::string());
EXPECT_EQ(JoinStrings(StringList({"a"}), 0, 0, ','), "");
EXPECT_EQ(JoinStrings(StringList({"a"}), 0, 1, ','), "a");
EXPECT_EQ(JoinStrings(StringList({"a"}), 0, 2, ','), "a");
EXPECT_EQ(JoinStrings(StringList({"a"}), 1, 0, ','), "");
EXPECT_EQ(JoinStrings(StringList({"a", "b", "c"}), 0, 1, ','), "a");
EXPECT_EQ(JoinStrings(StringList({"a", "b", "c"}), 1, 2, ','), "b");
EXPECT_EQ(JoinStrings(StringList({"a", "b", "c"}), 2, 3, ','), "c");
EXPECT_EQ(JoinStrings(StringList({"a", "b", "c"}), 0, 2, ','), "a,b");
EXPECT_EQ(JoinStrings(StringList({"a", "b", "c"}), 0, 3, ','), "a,b,c");
EXPECT_EQ(JoinStrings(StringList({"a", "b", "c"}), 1, 3, ','), "b,c");
EXPECT_EQ(JoinStrings(StringList({"a", "b", "c"}), 0, 10, ','), "a,b,c");
}
TEST(UtilTest, HumanBytes) {
EXPECT_EQ(HumanBytes(42), "42 bytes");
EXPECT_EQ(HumanBytes(42ull << 10), "42 KB");
EXPECT_EQ(HumanBytes(42ull << 20), "42 MB");
EXPECT_EQ(HumanBytes(42ull << 30), "42 GB");
EXPECT_EQ(HumanBytes(42ull << 40), "42 TB");
EXPECT_EQ(HumanBytes(42ull << 50), "42 PB");
EXPECT_EQ(HumanBytes(42.5 * 1024), "42 KB");
EXPECT_EQ(HumanBytes(42.5 * 1024, 0), "42 KB");
EXPECT_EQ(HumanBytes(42.5 * 1024, 1), "42.5 KB");
EXPECT_EQ(HumanBytes(42.5 * 1024, 2), "42.50 KB");
// Special case: values returned in byte and the precision is ignored, as it
// does not make sense for human-readable bytes to be fractional.
EXPECT_EQ(HumanBytes(42.5, 2), "42 bytes");
}
TEST(UtilTest, HumanDuration) {
EXPECT_EQ(HumanDuration(absl::Duration()), "00:00");
EXPECT_EQ(HumanDuration(absl::Seconds(5)), "00:05");
EXPECT_EQ(HumanDuration(absl::Seconds(59)), "00:59");
EXPECT_EQ(HumanDuration(absl::Seconds(359)), "05:59");
EXPECT_EQ(HumanDuration(absl::Seconds(12 * 60 + 34)), "12:34");
EXPECT_EQ(HumanDuration(absl::Minutes(2)), "02:00");
EXPECT_EQ(HumanDuration(absl::Minutes(42)), "42:00");
EXPECT_EQ(HumanDuration(absl::Minutes(123)), "123:00");
}
TEST(UtilTest, FinalSetter) {
int i = 0;
{
FinalSetter<int> fs(&i, 42);
EXPECT_EQ(i, 0);
}
EXPECT_EQ(i, 42);
std::string s = "foo";
{
FinalSetter<std::string> fs(&s, "bar");
EXPECT_EQ(s, "foo");
}
EXPECT_EQ(s, "bar");
}
TEST(UtilTest, GenerateUniqueId) {
std::string id = Util::GenerateUniqueId();
EXPECT_EQ(id.size(), 36);
for (int i = 0; i < 36; ++i) {
// Check that dashes are on correct positions.
if (i == 8 || i == 13 || i == 18 || i == 23) {
EXPECT_EQ(id[i], '-');
} else {
EXPECT_TRUE(id[i] >= '0' && id[i] <= '9' || id[i] >= 'A' && id[i] <= 'F');
}
}
EXPECT_NE(Util::GenerateUniqueId(), id);
}
// Can't cover Util::WaitForDebugger(), obviously.
} // namespace
} // namespace cdc_ft