Releasing the former Stadia file transfer tools

The tools allow efficient and fast synchronization of large directory trees from a Windows workstation to a Linux target machine. cdc_rsync* support efficient copy of files by using content-defined chunking (CDC) to identify chunks within files that can be reused. asset_stream_manager + cdc_fuse_fs support efficient streaming of a local directory to a remote virtual file system based on FUSE. It also employs CDC to identify and reuse unchanged data chunks.
2026-01-30 10:35:37 +02:00 · 2022-10-07 10:47:04 +02:00
commit 4326e972ac
364 changed files with 49410 additions and 0 deletions
--- a/third_party/absl
+++ b/third_party/absl
--- a/third_party/blake3/BUILD.bazel
+++ b/third_party/blake3/BUILD.bazel
@@ -0,0 +1,49 @@
+# Description:
+# BLAKE3 is a very fast cryptographic hash function, see README.md for details.
+#
+
+licenses(["unencumbered"])  # Creative Commons CC0
+
+package(default_visibility = ["//visibility:public"])
+
+exports_files(["LICENSE"])
+
+config_setting(
+    name = "windows",
+    values = {
+        "cpu": "x64_windows",
+    },
+)
+
+cc_library(
+    name = "blake3",
+    srcs = [
+        "c/blake3.c",
+        "c/blake3_dispatch.c",
+        "c/blake3_portable.c",
+    ] + select({
+        ":windows": [
+            "c/blake3_avx2_x86-64_windows_msvc.asm",
+            "c/blake3_avx512_x86-64_windows_msvc.asm",
+            "c/blake3_sse2_x86-64_windows_msvc.asm",
+            "c/blake3_sse41_x86-64_windows_msvc.asm",
+        ],
+        "//conditions:default": [
+            "c/blake3_avx2_x86-64_unix.S",
+            "c/blake3_avx512_x86-64_unix.S",
+            "c/blake3_sse2_x86-64_unix.S",
+            "c/blake3_sse41_x86-64_unix.S",
+        ],
+    }),
+    hdrs = [
+        "c/blake3.h",
+        "c/blake3_impl.h",
+    ],
+    includes = ["c"],
+)
+
+cc_binary(
+    name = "example",
+    srcs = ["c/example.c"],
+    deps = [":blake3"],
+)
--- a/third_party/dirent/BUILD.bazel
+++ b/third_party/dirent/BUILD.bazel
@@ -0,0 +1,15 @@
+# Description:
+# This project provides Linux compatible Dirent interface for Microsoft Windows.
+#
+
+licenses(["notice"])  # MIT license
+
+exports_files(["LICENSE"])
+
+package(default_visibility = ["//visibility:public"])
+
+cc_library(
+    name = "dirent",
+    hdrs = ["include/dirent.h"],
+    includes = ["include"],
+)
--- a/third_party/fuse/BUILD
+++ b/third_party/fuse/BUILD
@@ -0,0 +1,134 @@
+# Yeti's custom BUILD file
+package(default_visibility = ["//visibility:public"])
+
+licenses(["restricted"])  # GPL (binary), LGPL (library)
+
+exports_files(["COPYING"])
+
+# ------------------------------------------------------------------------------
+# Public libraries
+# ------------------------------------------------------------------------------
+
+DEFINES = [
+    "_FILE_OFFSET_BITS=64",
+    "FUSE_USE_VERSION=26",
+]
+
+COPTS = [
+    "-DHAVE_CONFIG_H=1",
+    "-DFUSERMOUNT_DIR=\\\"$(BINDIR)\\\"",
+    "-D_REENTRANT",
+    "-pthread",
+    "-fno-strict-aliasing",
+    "-Wno-use-after-free",  # Looks like a false positive.
+    "-iquote",
+    "third_party/fuse",
+    "-fvisibility=default",  # override -fvisibility=hidden from Yeti toolchain
+]
+
+cc_library(
+    name = "fuse_shared",
+    srcs = [":libfuse.so"],
+    copts = COPTS,
+    deps = [":fuse_headers"],
+)
+
+cc_binary(
+    name = "libfuse.so",
+    linkopts = ["-Wl,-soname,libfuse.so"],
+    linkshared = 1,
+    linkstatic = 0,
+    deps = ["fuse_internal"],
+)
+
+cc_library(
+    name = "fuse_internal",
+    srcs = [
+        "lib/buffer.c",
+        "lib/cuse_lowlevel.c",
+        "lib/fuse.c",
+        "lib/fuse_i.h",
+        "lib/fuse_kern_chan.c",
+        "lib/fuse_loop.c",
+        "lib/fuse_loop_mt.c",
+        "lib/fuse_lowlevel.c",
+        "lib/fuse_misc.h",
+        "lib/fuse_mt.c",
+        "lib/fuse_opt.c",
+        "lib/fuse_session.c",
+        "lib/fuse_signals.c",
+        "lib/helper.c",
+        "lib/modules/iconv.c",
+        "lib/modules/subdir.c",
+        "lib/mount.c",
+        "lib/mount_util.c",
+        "lib/mount_util.h",
+    ],
+    copts = COPTS,
+    linkopts = [
+        "-lpthread",
+        "-ldl",
+    ],
+    visibility = ["//visibility:private"],
+    deps = [":fuse_headers"],
+    alwayslink = 1,
+    linkstatic = 1,  # Required to make symbols show up in libfuse.so above.
+)
+
+cc_library(
+    name = "fuse_headers",
+    hdrs = glob(["include/*.h"]) + ["include/fuse/fuse.h"],
+    copts = COPTS,
+    defines = DEFINES,
+    includes = ["include"],
+    visibility = ["//visibility:private"],
+    deps = [":fuse_config"],
+)
+
+cc_library(
+    name = "ulockmgr",
+    srcs = [
+        "lib/ulockmgr.c",
+    ],
+    hdrs = [
+        "include/ulockmgr.h",
+    ],
+    copts = COPTS,
+    defines = DEFINES,
+    includes = ["include"],
+    linkopts = [
+        "-lpthread",
+        "-ldl",
+    ],
+)
+
+# ------------------------------------------------------------------------------
+# Genrules
+# ------------------------------------------------------------------------------
+
+# A crude hack to make #include <fuse/fuse.h> work.
+genrule(
+    name = "fuse_fuse_h",
+    outs = ["include/fuse/fuse.h"],
+    cmd = "echo '#include <fuse.h>' > $@",
+    visibility = ["//visibility:private"],
+)
+
+genrule(
+    name = "config_h",
+    srcs = ["@//third_party/fuse:linux_config"],
+    outs = ["config.h"],
+    cmd = "cp $< $@",
+    visibility = ["//visibility:private"],
+)
+
+cc_library(
+    name = "fuse_config",
+    srcs = ["config.h"],
+    includes = ["."],
+)
+
+filegroup(
+    name = "linux_config",
+    srcs = ["config.h.linux"],
+)
--- a/third_party/fuse/config.h.linux
+++ b/third_party/fuse/config.h.linux
@@ -0,0 +1,92 @@
+/* Define to 1 if you have the <dlfcn.h> header file. */
+#define HAVE_DLFCN_H 1
+
+/* Define to 1 if you have the `fdatasync' function. */
+#define HAVE_FDATASYNC 1
+
+/* Define to 1 if you have the `fork' function. */
+#define HAVE_FORK 1
+
+/* Define if you have the iconv() function and it works. */
+#define HAVE_ICONV 1
+
+/* Define to 1 if you have the <inttypes.h> header file. */
+#define HAVE_INTTYPES_H 1
+
+/* Define to 1 if you have the <memory.h> header file. */
+#define HAVE_MEMORY_H 1
+
+/* Define to 1 if you have the `posix_fallocate' function. */
+#define HAVE_POSIX_FALLOCATE 1
+
+/* Define to 1 if you have the `setxattr' function. */
+#define HAVE_SETXATTR 1
+
+/* Define to 1 if you have the `splice' function. */
+#define HAVE_SPLICE 1
+
+/* Define to 1 if you have the <stdint.h> header file. */
+#define HAVE_STDINT_H 1
+
+/* Define to 1 if you have the <stdlib.h> header file. */
+#define HAVE_STDLIB_H 1
+
+/* Define to 1 if you have the <strings.h> header file. */
+#define HAVE_STRINGS_H 1
+
+/* Define to 1 if you have the <string.h> header file. */
+#define HAVE_STRING_H 1
+
+/* Define to 1 if `st_atim' is a member of `struct stat'. */
+#define HAVE_STRUCT_STAT_ST_ATIM 1
+
+/* Define to 1 if `st_atimespec' is a member of `struct stat'. */
+/* #undef HAVE_STRUCT_STAT_ST_ATIMESPEC */
+
+/* Define to 1 if you have the <sys/stat.h> header file. */
+#define HAVE_SYS_STAT_H 1
+
+/* Define to 1 if you have the <sys/types.h> header file. */
+#define HAVE_SYS_TYPES_H 1
+
+/* Define to 1 if you have the <unistd.h> header file. */
+#define HAVE_UNISTD_H 1
+
+/* Define to 1 if you have the `utimensat' function. */
+#define HAVE_UTIMENSAT 1
+
+/* Define to 1 if you have the `vmsplice' function. */
+#define HAVE_VMSPLICE 1
+
+/* Define as const if the declaration of iconv() needs const. */
+#define ICONV_CONST 1
+
+/* Don't update /etc/mtab */
+/* #undef IGNORE_MTAB */
+
+/* Name of package */
+#define PACKAGE "fuse"
+
+/* Define to the address where bug reports for this package should be sent. */
+#define PACKAGE_BUGREPORT "https://github.com/libfuse/libfuse"
+
+/* Define to the full name of this package. */
+#define PACKAGE_NAME "fuse"
+
+/* Define to the full name and version of this package. */
+#define PACKAGE_STRING "fuse 2.9.7"
+
+/* Define to the one symbol short name of this package. */
+#define PACKAGE_TARNAME "fuse"
+
+/* Define to the home page for this package. */
+#define PACKAGE_URL "https://github.com/libfuse/libfuse"
+
+/* Define to the version of this package. */
+#define PACKAGE_VERSION "2.9.7"
+
+/* Define to 1 if you have the ANSI C header files. */
+#define STDC_HEADERS 1
+
+/* Version number of package */
+#define VERSION "2.9.7"
--- a/third_party/fuse/disable_symbol_versioning.patch
+++ b/third_party/fuse/disable_symbol_versioning.patch
@@ -0,0 +1,14 @@
+The linker doesn't seem to like symbol versioning.
+
+--- a/lib/fuse_misc.h
+++ b/lib/fuse_misc.h
+@@ -15,7 +15,8 @@
+     - not supported on MacOSX (in MachO binary format)
+ */
+ #if (!defined(__UCLIBC__) && !defined(__APPLE__))
+-#define FUSE_SYMVER(x) __asm__(x)
+//#define FUSE_SYMVER(x) __asm__(x)
+#define FUSE_SYMVER(x)
+ #else
+ #define FUSE_SYMVER(x)
+ #endif
--- a/third_party/googletest
+++ b/third_party/googletest
--- a/third_party/grpc
+++ b/third_party/grpc
--- a/third_party/protobuf
+++ b/third_party/protobuf
--- a/third_party/zstd/BUILD.bazel
+++ b/third_party/zstd/BUILD.bazel
@@ -0,0 +1,99 @@
+package(default_visibility = ["//visibility:public"])
+
+licenses(["notice"])
+
+COMMON_DEFINES = ["ZSTD_MULTITHREAD"]
+
+cc_library(
+    name = "zstd",
+    srcs = [
+        "lib/common/bitstream.h",
+        "lib/common/compiler.h",
+        "lib/common/cpu.h",
+        "lib/common/debug.c",
+        "lib/common/debug.h",
+        "lib/common/entropy_common.c",
+        "lib/common/error_private.c",
+        "lib/common/error_private.h",
+        "lib/common/fse.h",
+        "lib/common/fse_decompress.c",
+        "lib/common/huf.h",
+        "lib/common/mem.h",
+        "lib/common/pool.c",
+        "lib/common/pool.h",
+        "lib/common/portability_macros.h",
+        "lib/common/threading.c",
+        "lib/common/threading.h",
+        "lib/common/xxhash.c",
+        "lib/common/xxhash.h",
+        "lib/common/zstd_common.c",
+        "lib/common/zstd_deps.h",
+        "lib/common/zstd_internal.h",
+        "lib/common/zstd_trace.h",
+        "lib/compress/clevels.h",
+        "lib/compress/fse_compress.c",
+        "lib/compress/hist.c",
+        "lib/compress/hist.h",
+        "lib/compress/huf_compress.c",
+        "lib/compress/zstd_compress.c",
+        "lib/compress/zstd_compress_internal.h",
+        "lib/compress/zstd_compress_literals.c",
+        "lib/compress/zstd_compress_literals.h",
+        "lib/compress/zstd_compress_sequences.c",
+        "lib/compress/zstd_compress_sequences.h",
+        "lib/compress/zstd_compress_superblock.c",
+        "lib/compress/zstd_compress_superblock.h",
+        "lib/compress/zstd_cwksp.h",
+        "lib/compress/zstd_double_fast.c",
+        "lib/compress/zstd_double_fast.h",
+        "lib/compress/zstd_fast.c",
+        "lib/compress/zstd_fast.h",
+        "lib/compress/zstd_lazy.c",
+        "lib/compress/zstd_lazy.h",
+        "lib/compress/zstd_ldm.c",
+        "lib/compress/zstd_ldm.h",
+        "lib/compress/zstd_ldm_geartab.h",
+        "lib/compress/zstd_opt.c",
+        "lib/compress/zstd_opt.h",
+        "lib/compress/zstdmt_compress.c",
+        "lib/compress/zstdmt_compress.h",
+        "lib/decompress/huf_decompress.c",
+        "lib/decompress/zstd_ddict.c",
+        "lib/decompress/zstd_ddict.h",
+        "lib/decompress/zstd_decompress.c",
+        "lib/decompress/zstd_decompress_block.c",
+        "lib/decompress/zstd_decompress_block.h",
+        "lib/decompress/zstd_decompress_internal.h",
+        "lib/dictBuilder/cover.c",
+        "lib/dictBuilder/cover.h",
+        "lib/dictBuilder/divsufsort.c",
+        "lib/dictBuilder/divsufsort.h",
+        "lib/dictBuilder/fastcover.c",
+        "lib/dictBuilder/zdict.c",
+        "lib/legacy/zstd_legacy.h",
+        "lib/legacy/zstd_v01.c",
+        "lib/legacy/zstd_v01.h",
+        "lib/legacy/zstd_v02.c",
+        "lib/legacy/zstd_v02.h",
+        "lib/legacy/zstd_v03.c",
+        "lib/legacy/zstd_v03.h",
+        "lib/legacy/zstd_v04.c",
+        "lib/legacy/zstd_v04.h",
+        "lib/legacy/zstd_v05.c",
+        "lib/legacy/zstd_v05.h",
+        "lib/legacy/zstd_v06.c",
+        "lib/legacy/zstd_v06.h",
+        "lib/legacy/zstd_v07.c",
+        "lib/legacy/zstd_v07.h",
+    ] + select({
+        "@//tools:windows": [],
+        "//conditions:default": ["lib/decompress/huf_decompress_amd64.S"],
+    }),
+    hdrs = [
+        "lib/zdict.h",
+        "lib/zstd.h",
+        "lib/zstd_errors.h",
+    ],
+    defines = COMMON_DEFINES,
+    includes = ["lib"],
+)