Releasing the former Stadia file transfer tools

The tools allow efficient and fast synchronization of large directory
trees from a Windows workstation to a Linux target machine.

cdc_rsync* support efficient copy of files by using content-defined
chunking (CDC) to identify chunks within files that can be reused.

asset_stream_manager + cdc_fuse_fs support efficient streaming of a
local directory to a remote virtual file system based on FUSE. It also
employs CDC to identify and reuse unchanged data chunks.
This commit is contained in:
Christian Schneider
2022-10-07 10:47:04 +02:00
commit 4326e972ac
364 changed files with 49410 additions and 0 deletions

49
metrics/BUILD Normal file
View File

@@ -0,0 +1,49 @@
package(default_visibility = ["//:__subpackages__"])
cc_library(
name = "metrics",
srcs = [
"metrics.cc",
],
hdrs = [
"metrics.h",
],
deps = [
":enums",
":messages",
"//common:log",
"@com_google_absl//absl/status",
"@com_google_absl//absl/status:statusor",
"@com_google_absl//absl/strings:str_format",
],
)
cc_library(
name = "enums",
hdrs = ["enums.h"],
)
cc_library(
name = "messages",
srcs = ["messages.cc"],
hdrs = ["messages.h"],
deps = [
":enums",
"@com_google_absl//absl/status",
],
)
cc_test(
name = "messages_test",
srcs = ["messages_test.cc"],
deps = [
":messages",
"@com_google_googletest//:gtest",
"@com_google_googletest//:gtest_main",
],
)
filegroup(
name = "all_test_sources",
srcs = glob(["*_test.cc"]),
)

63
metrics/enums.h Normal file
View File

@@ -0,0 +1,63 @@
/*
* Copyright 2022 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef METRICS_ENUMS_H_
#define METRICS_ENUMS_H_
namespace cdc_ft {
namespace metrics {
// Event types logged by file transfer component.
enum class EventType {
kSessionHeartBeat,
kSessionStart,
kMultiSessionStart,
kSessionEnd,
kMultiSessionEnd,
kManifestUpdated
};
// Manifest update trigger.
enum class UpdateTrigger {
// Initial ManifestUpdater::UpdateAll() call
// at the start of a MultiSession.
kInitUpdateAll,
// ManifestUpdater::UpdateAll() call during a running MultiSession
// if the streamed folder is recreated.
kRunningUpdateAll,
// ManifestUpdater::Update() call to incrementally update modified files.
kRegularUpdate
};
// Session start status, which describes at which point
// of execution it failed.
enum class SessionStartStatus {
kOk,
kParseInstanceNameError,
kInvalidDirError,
kRestartSessionError,
kStopSessionError,
kCreateMultiSessionError,
kStartSessionError
};
// Where a request originated from.
enum class RequestOrigin { kUnknown, kCli, kPartnerPortal };
} // namespace metrics
} // namespace cdc_ft
#endif // METRICS_ENUMS_H_

101
metrics/messages.cc Normal file
View File

@@ -0,0 +1,101 @@
// Copyright 2022 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "metrics/messages.h"
namespace cdc_ft {
namespace metrics {
namespace {
template <typename T>
inline bool Eq(const std::unique_ptr<T>& lhs, const std::unique_ptr<T>& rhs) {
return (!lhs && !rhs) || (lhs && rhs && *lhs == *rhs);
}
} // namespace
bool operator==(const ManifestUpdateData& lhs, const ManifestUpdateData& rhs) {
return lhs.total_assets_added_or_updated ==
rhs.total_assets_added_or_updated &&
lhs.total_files_added_or_updated == rhs.total_files_added_or_updated &&
lhs.total_files_failed == rhs.total_files_failed &&
lhs.total_assets_deleted == rhs.total_assets_deleted &&
lhs.total_chunks == rhs.total_chunks &&
lhs.total_processed_bytes == rhs.total_processed_bytes &&
lhs.local_duration_ms == rhs.local_duration_ms &&
lhs.trigger == rhs.trigger && lhs.status == rhs.status;
}
bool operator!=(const ManifestUpdateData& lhs, const ManifestUpdateData& rhs) {
return !(lhs == rhs);
}
bool operator==(const SessionStartData& lhs, const SessionStartData& rhs) {
return lhs.concurrent_session_count == rhs.concurrent_session_count &&
lhs.absl_status == rhs.absl_status && lhs.status == rhs.status &&
lhs.origin == rhs.origin;
}
bool operator!=(const SessionStartData& lhs, const SessionStartData& rhs) {
return !(lhs == rhs);
}
bool operator==(const MultiSessionStartData& lhs,
const MultiSessionStartData& rhs) {
return lhs.file_count == rhs.file_count && lhs.byte_count == rhs.byte_count &&
lhs.chunk_count == rhs.chunk_count &&
lhs.min_chunk_size == rhs.min_chunk_size &&
lhs.avg_chunk_size == rhs.avg_chunk_size &&
lhs.max_chunk_size == rhs.max_chunk_size;
}
bool operator!=(const MultiSessionStartData& lhs,
const MultiSessionStartData& rhs) {
return !(lhs == rhs);
}
bool operator==(const SessionData& lhs, const SessionData& rhs) {
return lhs.chunk_count == rhs.chunk_count && lhs.byte_count == rhs.byte_count;
}
bool operator!=(const SessionData& lhs, const SessionData& rhs) {
return !(lhs == rhs);
}
bool operator==(const AssetStreamingManagerData& lhs,
const AssetStreamingManagerData& rhs) {
return Eq(lhs.session_data, rhs.session_data) &&
Eq(lhs.manifest_update_data, rhs.manifest_update_data) &&
Eq(lhs.session_start_data, rhs.session_start_data) &&
Eq(lhs.multi_session_start_data, rhs.multi_session_start_data) &&
lhs.multisession_id == rhs.multisession_id &&
lhs.session_id == rhs.session_id;
}
bool operator!=(const AssetStreamingManagerData& lhs,
const AssetStreamingManagerData& rhs) {
return !(lhs == rhs);
}
bool operator==(const DeveloperLogEvent& lhs, const DeveloperLogEvent& rhs) {
return lhs.project_id == rhs.project_id &&
lhs.organization_id == rhs.organization_id &&
Eq(lhs.as_manager_data, rhs.as_manager_data);
}
bool operator!=(const DeveloperLogEvent& lhs, const DeveloperLogEvent& rhs) {
return !(lhs == rhs);
}
} // namespace metrics
} // namespace cdc_ft

157
metrics/messages.h Normal file
View File

@@ -0,0 +1,157 @@
/*
* Copyright 2022 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef METRICS_MESSAGES_H_
#define METRICS_MESSAGES_H_
#include "absl/status/status.h"
#include "metrics/enums.h"
namespace cdc_ft {
namespace metrics {
// Holds manifest update stats, status and trigger.
struct ManifestUpdateData {
// Manifest update stats.
uint64_t total_assets_added_or_updated;
uint64_t total_files_added_or_updated;
uint64_t total_files_failed;
uint64_t total_assets_deleted;
uint64_t total_chunks;
uint64_t total_processed_bytes;
// Duration of local manifest update.
// This doesn't include any workstation-gamelet communication time.
uint64_t local_duration_ms;
UpdateTrigger trigger;
absl::StatusCode status;
};
bool operator==(const ManifestUpdateData& lhs, const ManifestUpdateData& rhs);
bool operator!=(const ManifestUpdateData& lhs, const ManifestUpdateData& rhs);
// Contains data which is recorded on asset streaming session start.
struct SessionStartData {
// Current number of simultaneous sessions in multisession
// including the newly started one.
uint32_t concurrent_session_count;
// Session abseil start status.
absl::StatusCode absl_status;
// Session start status, which describes at which point
// of execution it failed.
SessionStartStatus status;
// Where the StartSession request originated from.
RequestOrigin origin;
};
bool operator==(const SessionStartData& lhs, const SessionStartData& rhs);
bool operator!=(const SessionStartData& lhs, const SessionStartData& rhs);
// Contains data which is recorded on asset streaming multisession start.
struct MultiSessionStartData {
// Initial number of files in the streamed directory.
uint32_t file_count;
// Initial number of bytes in the streamed directory.
uint64_t byte_count;
// Initial total chunk count.
uint64_t chunk_count;
// Minimum chunk size CDC parameter.
uint64_t min_chunk_size;
// Average chunk size CDC parameter.
uint64_t avg_chunk_size;
// Maximum chunk size CDC parameter.
uint64_t max_chunk_size;
};
bool operator==(const MultiSessionStartData& lhs,
const MultiSessionStartData& rhs);
bool operator!=(const MultiSessionStartData& lhs,
const MultiSessionStartData& rhs);
// Session Heartbeat data.
// It is sent every 5 minutes in case data has been changed
// and at the end of a session. The counters are cumulative, so in order
// to calculate total size of data sent during a particular session
// the value of the last event should be queried.
struct SessionData {
// Number of chunks sent during the session so far.
uint64_t chunk_count;
// Number of bytes sent during the session so far.
uint64_t byte_count;
};
bool operator==(const SessionData& lhs, const SessionData& rhs);
bool operator!=(const SessionData& lhs, const SessionData& rhs);
struct AssetStreamingManagerData {
// Session Heartbeat data.
std::unique_ptr<SessionData> session_data;
// Holds manifest update stats, status and trigger.
std::unique_ptr<ManifestUpdateData> manifest_update_data;
// Contains data which is recorded on asset streaming session start.
std::unique_ptr<SessionStartData> session_start_data;
// Contains data which is recorded on asset streaming multisession start.
std::unique_ptr<MultiSessionStartData> multi_session_start_data;
// Randomly-generated whenever asset streaming multisession is created.
std::string multisession_id;
// Randomly-generated whenever asset streaming session is created.
std::string session_id;
};
bool operator==(const AssetStreamingManagerData& lhs,
const AssetStreamingManagerData& rhs);
bool operator!=(const AssetStreamingManagerData& lhs,
const AssetStreamingManagerData& rhs);
// Contains developer log events, sent by GGP tools running on developer
// workstations.
struct DeveloperLogEvent {
DeveloperLogEvent() = default;
DeveloperLogEvent(DeveloperLogEvent&& other)
: project_id(std::move(other.project_id)),
organization_id(std::move(other.organization_id)),
as_manager_data(std::move(other.as_manager_data)) {}
DeveloperLogEvent& operator=(DeveloperLogEvent&& other) {
project_id = std::move(other.project_id);
organization_id = std::move(other.organization_id);
as_manager_data = std::move(other.as_manager_data);
return *this;
}
// GGP Project ID that is selected for the current session.
std::string project_id;
// GGP Publisher/Organization ID that is selected for the current session.
std::string organization_id;
// Describes asset streaming manager data.
std::unique_ptr<AssetStreamingManagerData> as_manager_data;
};
bool operator==(const DeveloperLogEvent& lhs, const DeveloperLogEvent& rhs);
bool operator!=(const DeveloperLogEvent& lhs, const DeveloperLogEvent& rhs);
} // namespace metrics
} // namespace cdc_ft
#endif // METRICS_MESSAGES_H_

222
metrics/messages_test.cc Normal file
View File

@@ -0,0 +1,222 @@
// Copyright 2022 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "metrics/messages.h"
#include "absl/status/status.h"
#include "gtest/gtest.h"
namespace cdc_ft {
namespace metrics {
namespace {
class MessagesTest : public ::testing::Test {
protected:
ManifestUpdateData GetManifestUpdateData() {
ManifestUpdateData data;
data.total_assets_added_or_updated = 1;
data.total_files_added_or_updated = 2;
data.total_files_failed = 3;
data.total_assets_deleted = 4;
data.total_chunks = 5;
data.total_processed_bytes = 6;
data.local_duration_ms = 7;
data.trigger = UpdateTrigger::kRegularUpdate;
data.status = absl::StatusCode::kResourceExhausted;
return data;
}
SessionStartData GetSessionStartData() {
SessionStartData data;
data.absl_status = absl::StatusCode::kInvalidArgument;
data.status = SessionStartStatus::kRestartSessionError;
data.concurrent_session_count = 1;
data.origin = RequestOrigin::kPartnerPortal;
return data;
}
MultiSessionStartData GetMultiSessionStartData() {
MultiSessionStartData data;
data.byte_count = 1;
data.chunk_count = 2;
data.file_count = 3;
data.min_chunk_size = 4;
data.avg_chunk_size = 5;
data.max_chunk_size = 6;
return data;
}
SessionData GetSessionData() {
SessionData data;
data.byte_count = 1;
data.chunk_count = 2;
return data;
}
AssetStreamingManagerData GetAssetStreamingManagerData() {
AssetStreamingManagerData data;
data.session_id = "id1";
data.multisession_id = "id2";
data.manifest_update_data =
std::make_unique<ManifestUpdateData>(GetManifestUpdateData());
data.multi_session_start_data =
std::make_unique<MultiSessionStartData>(GetMultiSessionStartData());
data.session_data = std::make_unique<SessionData>(GetSessionData());
data.session_start_data =
std::make_unique<SessionStartData>(GetSessionStartData());
return data;
}
DeveloperLogEvent GetDeveloperLogEvent() {
DeveloperLogEvent data;
data.project_id = "id1";
data.organization_id = "id2";
data.as_manager_data = std::make_unique<AssetStreamingManagerData>(
GetAssetStreamingManagerData());
return data;
}
};
TEST_F(MessagesTest, ManifestUpdateDataEqual) {
EXPECT_EQ(GetManifestUpdateData(), GetManifestUpdateData());
ManifestUpdateData data = GetManifestUpdateData();
data.total_assets_added_or_updated = 11;
EXPECT_NE(data, GetManifestUpdateData());
data = GetManifestUpdateData();
data.total_files_added_or_updated = 12;
EXPECT_NE(data, GetManifestUpdateData());
data = GetManifestUpdateData();
data.total_files_failed = 13;
EXPECT_NE(data, GetManifestUpdateData());
data = GetManifestUpdateData();
data.total_assets_deleted = 14;
EXPECT_NE(data, GetManifestUpdateData());
data = GetManifestUpdateData();
data.total_chunks = 15;
EXPECT_NE(data, GetManifestUpdateData());
data = GetManifestUpdateData();
data.total_processed_bytes = 16;
EXPECT_NE(data, GetManifestUpdateData());
data = GetManifestUpdateData();
data.local_duration_ms = 17;
EXPECT_NE(data, GetManifestUpdateData());
data = GetManifestUpdateData();
data.trigger = UpdateTrigger::kRunningUpdateAll;
EXPECT_NE(data, GetManifestUpdateData());
data = GetManifestUpdateData();
data.status = absl::StatusCode::kFailedPrecondition;
EXPECT_NE(data, GetManifestUpdateData());
}
TEST_F(MessagesTest, SessionStartDataEqual) {
EXPECT_EQ(GetSessionStartData(), GetSessionStartData());
SessionStartData data = GetSessionStartData();
data.absl_status = absl::StatusCode::kUnauthenticated;
EXPECT_NE(data, GetSessionStartData());
data = GetSessionStartData();
data.status = SessionStartStatus::kStopSessionError;
EXPECT_NE(data, GetSessionStartData());
data = GetSessionStartData();
data.concurrent_session_count = 11;
EXPECT_NE(data, GetSessionStartData());
data = GetSessionStartData();
data.origin = RequestOrigin::kUnknown;
EXPECT_NE(data, GetSessionStartData());
}
TEST_F(MessagesTest, MultiSessionStartDataEqual) {
EXPECT_EQ(GetMultiSessionStartData(), GetMultiSessionStartData());
MultiSessionStartData data = GetMultiSessionStartData();
data.byte_count = 11;
EXPECT_NE(data, GetMultiSessionStartData());
data = GetMultiSessionStartData();
data.chunk_count = 12;
EXPECT_NE(data, GetMultiSessionStartData());
data = GetMultiSessionStartData();
data.file_count = 13;
EXPECT_NE(data, GetMultiSessionStartData());
data = GetMultiSessionStartData();
data.min_chunk_size = 14;
EXPECT_NE(data, GetMultiSessionStartData());
data = GetMultiSessionStartData();
data.avg_chunk_size = 15;
EXPECT_NE(data, GetMultiSessionStartData());
data = GetMultiSessionStartData();
data.max_chunk_size = 16;
EXPECT_NE(data, GetMultiSessionStartData());
}
TEST_F(MessagesTest, SessionDataEqual) {
EXPECT_EQ(GetSessionData(), GetSessionData());
SessionData data = GetSessionData();
data.byte_count = 11;
EXPECT_NE(data, GetSessionData());
data = GetSessionData();
data.chunk_count = 12;
EXPECT_NE(data, GetSessionData());
}
TEST_F(MessagesTest, AssetStreamingManagerDataEqual) {
EXPECT_EQ(GetAssetStreamingManagerData(), GetAssetStreamingManagerData());
AssetStreamingManagerData data = GetAssetStreamingManagerData();
data.session_id = "id-11";
EXPECT_NE(data, GetAssetStreamingManagerData());
data = GetAssetStreamingManagerData();
data.multisession_id = "id-12";
EXPECT_NE(data, GetAssetStreamingManagerData());
data = GetAssetStreamingManagerData();
data.manifest_update_data = NULL;
EXPECT_NE(data, GetAssetStreamingManagerData());
data = GetAssetStreamingManagerData();
data.manifest_update_data->total_assets_added_or_updated = 11;
EXPECT_NE(data, GetAssetStreamingManagerData());
data = GetAssetStreamingManagerData();
data.multi_session_start_data = NULL;
EXPECT_NE(data, GetAssetStreamingManagerData());
data = GetAssetStreamingManagerData();
data.multi_session_start_data->byte_count = 11;
EXPECT_NE(data, GetAssetStreamingManagerData());
data = GetAssetStreamingManagerData();
data.session_data = NULL;
EXPECT_NE(data, GetAssetStreamingManagerData());
data = GetAssetStreamingManagerData();
data.session_data->byte_count = 11;
EXPECT_NE(data, GetAssetStreamingManagerData());
data = GetAssetStreamingManagerData();
data.session_start_data = NULL;
EXPECT_NE(data, GetAssetStreamingManagerData());
data = GetAssetStreamingManagerData();
data.session_start_data->absl_status = absl::StatusCode::kOutOfRange;
EXPECT_NE(data, GetAssetStreamingManagerData());
}
TEST_F(MessagesTest, DeveloperLogEventEqual) {
EXPECT_EQ(GetDeveloperLogEvent(), GetDeveloperLogEvent());
DeveloperLogEvent data = GetDeveloperLogEvent();
data.project_id = "id-11";
EXPECT_NE(data, GetDeveloperLogEvent());
data = GetDeveloperLogEvent();
data.organization_id = "id-12";
EXPECT_NE(data, GetDeveloperLogEvent());
data = GetDeveloperLogEvent();
data.as_manager_data = NULL;
EXPECT_NE(data, GetDeveloperLogEvent());
data = GetDeveloperLogEvent();
data.as_manager_data->session_data = NULL;
EXPECT_NE(data, GetDeveloperLogEvent());
}
} // namespace
} // namespace metrics
} // namespace cdc_ft

26
metrics/metrics.cc Normal file
View File

@@ -0,0 +1,26 @@
// Copyright 2022 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "metrics/metrics.h"
namespace cdc_ft {
namespace metrics {
MetricsService::MetricsService() {}
void MetricsService::RecordEvent(DeveloperLogEvent event,
EventType code) const {}
} // namespace metrics
} // namespace cdc_ft

37
metrics/metrics.h Normal file
View File

@@ -0,0 +1,37 @@
/*
* Copyright 2022 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef METRICS_METRICS_H_
#define METRICS_METRICS_H_
#include "metrics/enums.h"
#include "metrics/messages.h"
namespace cdc_ft {
namespace metrics {
class MetricsService {
public:
MetricsService();
// Sends a metrics event to metrics uploader.
virtual void RecordEvent(DeveloperLogEvent event, EventType code) const;
};
} // namespace metrics
} // namespace cdc_ft
#endif // METRICS_METRICS_H_