Releasing the former Stadia file transfer tools

The tools allow efficient and fast synchronization of large directory
trees from a Windows workstation to a Linux target machine.

cdc_rsync* support efficient copy of files by using content-defined
chunking (CDC) to identify chunks within files that can be reused.

asset_stream_manager + cdc_fuse_fs support efficient streaming of a
local directory to a remote virtual file system based on FUSE. It also
employs CDC to identify and reuse unchanged data chunks.
This commit is contained in:
Christian Schneider
2022-10-07 10:47:04 +02:00
commit 4326e972ac
364 changed files with 49410 additions and 0 deletions

12
absl_helper/BUILD Normal file
View File

@@ -0,0 +1,12 @@
package(default_visibility = ["//visibility:public"])
cc_library(
name = "jedec_size_flag",
srcs = ["jedec_size_flag.cc"],
hdrs = ["jedec_size_flag.h"],
deps = [
"@com_google_absl//absl/flags:flag",
"@com_google_absl//absl/flags:marshalling",
"@com_google_absl//absl/strings",
],
)

View File

@@ -0,0 +1,93 @@
// Copyright 2022 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "absl_helper/jedec_size_flag.h"
namespace cdc_ft {
namespace {
JedecUnit ToJedecUnit(char c) {
switch (c) {
case 'b':
case 'B':
return JedecUnit::Byte;
case 'k':
case 'K':
return JedecUnit::Kilo;
case 'm':
case 'M':
return JedecUnit::Mega;
case 'g':
case 'G':
return JedecUnit::Giga;
case 't':
case 'T':
return JedecUnit::Tera;
case 'p':
case 'P':
return JedecUnit::Peta;
default:
return JedecUnit::Unkown;
}
}
int LeftShiftAmount(JedecUnit unit) {
switch (unit) {
case JedecUnit::Kilo:
return 10;
case JedecUnit::Mega:
return 20;
case JedecUnit::Giga:
return 30;
case JedecUnit::Tera:
return 40;
case JedecUnit::Peta:
return 50;
default:
return 0;
}
}
} // namespace
bool AbslParseFlag(absl::string_view text, JedecSize* flag, std::string* err) {
if (text.empty()) return false;
JedecUnit unit = ToJedecUnit(text.back());
if (unit != JedecUnit::Unkown) {
text.remove_suffix(1);
} else {
// Are we dealing with a digit character?
if (text.back() >= '0' && text.back() <= '9') {
unit = JedecUnit::Byte;
} else {
*err =
"Supported size units are (B)yte, (K)ilo, (M)ega, (G)iga, (T)era, "
"(P)eta.";
return false;
}
}
// Try to parse a plain uint64_t value.
uint64_t size;
if (!absl::ParseFlag(text, &size, err)) {
return false;
}
flag->SetSize(size << LeftShiftAmount(unit));
return true;
}
std::string AbslUnparseFlag(const JedecSize& size) {
return absl::UnparseFlag(size.Size());
}
}; // namespace cdc_ft

View File

@@ -0,0 +1,61 @@
/*
* Copyright 2022 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef ABSL_HELPER_JEDEC_SIZE_FLAG_H_
#define ABSL_HELPER_JEDEC_SIZE_FLAG_H_
#include <string>
#include "absl/flags/flag.h"
#include "absl/flags/marshalling.h"
#include "absl/strings/string_view.h"
namespace cdc_ft {
// Supported JEDEC unit suffixes.
enum class JedecUnit : char {
Unkown = 0,
Byte = 'B', // optional
Kilo = 'K',
Mega = 'M',
Giga = 'G',
Tera = 'T',
Peta = 'P',
};
// This class parses flag arguments that represent human readable data sizes,
// such as 1024, 2K, 3M, 4G, or 5T.
//
// See https://en.wikipedia.org/wiki/JEDEC_memory_standards.
class JedecSize {
public:
explicit JedecSize(uint64_t size = 0) : size_(size) {}
uint64_t Size() const { return size_; }
void SetSize(uint64_t size) { size_ = size; }
private:
uint64_t size_;
};
// Abseil flags parser for JedecSize.
bool AbslParseFlag(absl::string_view text, JedecSize* flag, std::string* err);
// Abseil flags unparser for JedecSize.
std::string AbslUnparseFlag(const JedecSize& size);
}; // namespace cdc_ft
#endif // ABSL_HELPER_JEDEC_SIZE_FLAG_H_