Releasing the former Stadia file transfer tools

The tools allow efficient and fast synchronization of large directory
trees from a Windows workstation to a Linux target machine.

cdc_rsync* support efficient copy of files by using content-defined
chunking (CDC) to identify chunks within files that can be reused.

asset_stream_manager + cdc_fuse_fs support efficient streaming of a
local directory to a remote virtual file system based on FUSE. It also
employs CDC to identify and reuse unchanged data chunks.

cdc_rsync_server/.gitignore

@@ -0,0 +1,4 @@
+GGP/*
+generated_protos
+*.log
+*.user

cdc_rsync_server/BUILD

@@ -0,0 +1,158 @@
+package(default_visibility = [
+    "//:__subpackages__",
+])
+
+cc_library(
+    name = "file_deleter_and_sender",
+    srcs = ["file_deleter_and_sender.cc"],
+    hdrs = ["file_deleter_and_sender.h"],
+    deps = [
+        "//cdc_rsync/base:message_pump",
+        "//cdc_rsync/protos:messages_cc_proto",
+        "//common:path",
+        "//common:status",
+        "@com_google_absl//absl/status",
+    ],
+)
+
+cc_test(
+    name = "file_deleter_and_sender_test",
+    srcs = ["file_deleter_and_sender_test.cc"],
+    deps = [
+        ":file_deleter_and_sender",
+        "//cdc_rsync/base:fake_socket",
+        "//common:status_test_macros",
+        "//common:test_main",
+        "@com_google_googletest//:gtest",
+    ],
+)
+
+cc_library(
+    name = "file_finder",
+    srcs = ["file_finder.cc"],
+    hdrs = ["file_finder.h"],
+    deps = [
+        ":file_info",
+        "//common:path",
+        "//common:path_filter",
+        "//common:status",
+        "@com_google_absl//absl/status",
+    ],
+)
+
+cc_test(
+    name = "file_finder_test",
+    srcs = ["file_finder_test.cc"],
+    data = ["testdata/root.txt"] + glob(["testdata/file_finder/**"]),
+    deps = [
+        ":file_finder",
+        "//common:status_test_macros",
+        "//common:test_main",
+        "@com_google_googletest//:gtest",
+    ],
+)
+
+cc_library(
+    name = "file_diff_generator",
+    srcs = ["file_diff_generator.cc"],
+    hdrs = ["file_diff_generator.h"],
+    deps = [
+        ":file_info",
+        "//cdc_rsync/protos:messages_cc_proto",
+        "//common:log",
+        "//common:path",
+        "//common:util",
+    ],
+)
+
+cc_test(
+    name = "file_diff_generator_test",
+    srcs = ["file_diff_generator_test.cc"],
+    data = ["testdata/root.txt"] + glob(["testdata/file_diff_generator/**"]),
+    deps = [
+        ":file_diff_generator",
+        "//common:status_test_macros",
+        "//common:test_main",
+        "@com_google_googletest//:gtest",
+    ],
+)
+
+cc_binary(
+    name = "cdc_rsync_server",
+    srcs = [
+        "cdc_rsync_server.cc",
+        "cdc_rsync_server.h",
+        "main.cc",
+    ],
+    copts = select({
+        #":debug_build": ["-fstandalone-debug"],
+        "//conditions:default": [],
+    }),
+    deps = [
+        ":file_deleter_and_sender",
+        ":file_diff_generator",
+        ":file_finder",
+        ":file_info",
+        ":server_socket",
+        ":unzstd_stream",
+        "//cdc_rsync/base:cdc_interface",
+        "//cdc_rsync/base:message_pump",
+        "//cdc_rsync/base:server_exit_code",
+        "//common:clock",
+        "//common:gamelet_component",
+        "//common:log",
+        "//common:path_filter",
+        "//common:status",
+        "//common:stopwatch",
+        "//common:threadpool",
+        "//common:util",
+    ],
+)
+
+config_setting(
+    name = "debug_build",
+    values = {
+        "compilation_mode": "dbg",
+    },
+)
+
+cc_library(
+    name = "file_info",
+    hdrs = ["file_info.h"],
+)
+
+cc_library(
+    name = "server_socket",
+    srcs = ["server_socket.cc"],
+    hdrs = ["server_socket.h"],
+    target_compatible_with = ["@platforms//os:linux"],
+    deps = [
+        "//cdc_rsync/base:socket",
+        "//common:log",
+        "//common:status",
+        "@com_google_absl//absl/status",
+    ],
+)
+
+cc_library(
+    name = "unzstd_stream",
+    srcs = ["unzstd_stream.cc"],
+    hdrs = ["unzstd_stream.h"],
+    deps = [
+        "//cdc_rsync/base:message_pump",
+        "//cdc_rsync/base:socket",
+        "//common:status",
+        "@com_github_zstd//:zstd",
+        "@com_google_absl//absl/status",
+    ],
+)
+
+filegroup(
+    name = "all_test_sources",
+    srcs = glob(["*_test.cc"]),
+)
+
+filegroup(
+    name = "all_test_data",
+    srcs = glob(["testdata/**"]),
+)

cdc_rsync_server/cdc_rsync_server.cc

@@ -0,0 +1,758 @@
+// Copyright 2022 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "cdc_rsync_server/cdc_rsync_server.h"
+
+#include "absl/strings/str_format.h"
+#include "cdc_rsync/base/cdc_interface.h"
+#include "cdc_rsync/protos/messages.pb.h"
+#include "cdc_rsync_server/file_deleter_and_sender.h"
+#include "cdc_rsync_server/file_finder.h"
+#include "cdc_rsync_server/server_socket.h"
+#include "cdc_rsync_server/unzstd_stream.h"
+#include "common/log.h"
+#include "common/path.h"
+#include "common/status.h"
+#include "common/stopwatch.h"
+#include "common/threadpool.h"
+#include "common/util.h"
+
+namespace cdc_ft {
+
+namespace {
+
+// Suffix for the patched file created from the basis file and the diff.
+constexpr char kIntermediatePathSuffix[] = ".__cdc_rsync_temp__";
+
+uint16_t kExecutableBits =
+    path::MODE_IXUSR | path::MODE_IXGRP | path::MODE_IXOTH;
+
+// Background task that receives patch info for the base file at |base_filepath|
+// and writes the patched file to |target_filepath|. If |base_filepath| and
+// |target_filepath| match, writes an intermediate file and replaces
+// the file at |target_filepath| with the intermediate file when all data has
+// been received.
+class PatchTask : public Task {
+ public:
+  PatchTask(const std::string& base_filepath,
+            const std::string& target_filepath, const ChangedFileInfo& file,
+            CdcInterface* cdc)
+      : base_filepath_(base_filepath),
+        target_filepath_(target_filepath),
+        file_(file),
+        cdc_(cdc) {}
+
+  virtual ~PatchTask() = default;
+
+  const ChangedFileInfo& File() const { return file_; }
+
+  const absl::Status& Status() const { return status_; }
+
+  // Task:
+  void ThreadRun(IsCancelledPredicate is_cancelled) override {
+    bool need_intermediate_file = target_filepath_ == base_filepath_;
+    std::string patched_filepath =
+        need_intermediate_file ? base_filepath_ + kIntermediatePathSuffix
+                               : target_filepath_;
+
+    absl::StatusOr<FILE*> patched_file = path::OpenFile(patched_filepath, "wb");
+    if (!patched_file.ok()) {
+      status_ = patched_file.status();
+      return;
+    }
+
+    // Receive diff stream from server and apply.
+    bool is_executable = false;
+    status_ = cdc_->ReceiveDiffAndPatch(base_filepath_, *patched_file,
+                                        &is_executable);
+    fclose(*patched_file);
+    if (!status_.ok()) {
+      return;
+    }
+
+    // These bits are OR'ed on top of the mode bits.
+    uint16_t mode_or_bits = is_executable ? kExecutableBits : 0;
+
+    // Store mode from the original base path.
+    path::Stats stats;
+    status_ = GetStats(base_filepath_, &stats);
+    if (!status_.ok()) {
+      status_ =
+          WrapStatus(status_, "GetStats() failed for '%s'", base_filepath_);
+      return;
+    }
+
+    if (need_intermediate_file) {
+      // Replace |base_filepath_| (==|target_filepath_|) by the intermediate
+      // file |patched_filepath|.
+      status_ = path::ReplaceFile(target_filepath_, patched_filepath);
+      if (!status_.ok()) {
+        status_ = WrapStatus(status_, "ReplaceFile() for '%s' by '%s' failed",
+                             base_filepath_, patched_filepath);
+        return;
+      }
+    } else {
+      // An intermediate file is typically not needed when the base path is
+      // a file in a package. Since package files are read-only, we add the
+      // write bit, so that the file can be overwritten with the next sync.
+      mode_or_bits |= path::Mode::MODE_IWUSR;
+    }
+
+    // Restore mode, possibly adding executable bit and user write bit.
+    status_ = path::ChangeMode(target_filepath_, stats.mode | mode_or_bits);
+    if (!status_.ok()) {
+      status_ =
+          WrapStatus(status_, "ChangeMode() failed for '%s'", base_filepath_);
+      return;
+    }
+
+    status_ = path::SetFileTime(target_filepath_, file_.client_modified_time);
+  }
+
+ private:
+  std::string base_filepath_;
+  std::string target_filepath_;
+  ChangedFileInfo file_;
+  CdcInterface* cdc_;
+  absl::Status status_;
+};
+
+PathFilter::Rule::Type ToInternalType(
+    SetOptionsRequest::FilterRule::Type type) {
+  switch (type) {
+    case SetOptionsRequest::FilterRule::TYPE_INCLUDE:
+      return PathFilter::Rule::Type::kInclude;
+    case SetOptionsRequest::FilterRule::TYPE_EXCLUDE:
+      return PathFilter::Rule::Type::kExclude;
+    // Make compiler happy...
+    case SetOptionsRequest_FilterRule_Type_SetOptionsRequest_FilterRule_Type_INT_MIN_SENTINEL_DO_NOT_USE_:
+    case SetOptionsRequest_FilterRule_Type_SetOptionsRequest_FilterRule_Type_INT_MAX_SENTINEL_DO_NOT_USE_:
+      break;
+  }
+  assert(false);
+  return PathFilter::Rule::Type::kInclude;
+}
+
+}  // namespace
+
+GgpRsyncServer::GgpRsyncServer() = default;
+
+GgpRsyncServer::~GgpRsyncServer() {
+  message_pump_.reset();
+  socket_.reset();
+}
+
+bool GgpRsyncServer::CheckComponents(
+    const std::vector<GameletComponent>& components) {
+  // Components are expected to reside in the same dir as the executable.
+  std::string component_dir;
+  absl::Status status = path::GetExeDir(&component_dir);
+  if (!status.ok()) {
+    return false;
+  }
+
+  std::vector<GameletComponent> our_components;
+  status = GameletComponent::Get(
+      {path::Join(component_dir, "cdc_rsync_server")}, &our_components);
+  if (!status.ok() || components != our_components) {
+    return false;
+  }
+
+  return true;
+}
+
+absl::Status GgpRsyncServer::Run(int port) {
+  socket_ = std::make_unique<ServerSocket>();
+  absl::Status status = socket_->StartListening(port);
+  if (!status.ok()) {
+    return WrapStatus(status, "Failed to start listening on port %i", port);
+  }
+  LOG_INFO("cdc_rsync_server listening on port %i", port);
+
+  // This is the marker for the client, so it knows it can connect.
+  printf("Server is listening\n");
+  fflush(stdout);
+
+  status = socket_->WaitForConnection();
+  if (!status.ok()) {
+    return WrapStatus(status, "Failed to establish a connection");
+  }
+
+  message_pump_ = std::make_unique<MessagePump>(
+      socket_.get(),
+      [this](PacketType type) { Thread_OnPackageReceived(type); });
+  message_pump_->StartMessagePump();
+
+  LOG_INFO("Client connected. Starting to sync.");
+  status = Sync();
+  if (!status.ok()) {
+    socket_->ShutdownSendingEnd().IgnoreError();
+    return status;
+  }
+
+  LOG_INFO("Exiting cdc_rsync_server");
+  return absl::OkStatus();
+}
+
+absl::Status GgpRsyncServer::Sync() {
+  // First, the client sends us options, e.g. the |destination_| directory.
+  absl::Status status = HandleSetOptions();
+  if (!status.ok()) {
+    return WrapStatus(status, "Failed to receive options");
+  }
+
+  // Find all files in the |destination_| and |copy_dest_| (if set) directories.
+  status = FindFiles();
+  if (!status.ok()) {
+    return WrapStatus(status, "Failed to find files on instance");
+  }
+
+  // Get the list of all files that the client sends us.
+  status = HandleSendAllFiles();
+  if (!status.ok()) {
+    return WrapStatus(status, "Failed to receive client file info");
+  }
+
+  // Diff client and server files and send missing files to the client.
+  status = DiffFiles();
+  if (!status.ok()) {
+    return WrapStatus(status, "Failed to compute file difference");
+  }
+
+  // Delete files and directories not present on the client.
+  if (delete_) {
+    status = RemoveExtraneousFilesAndDirs();
+    if (!status.ok()) {
+      return WrapStatus(status, "Failed to delete files and directories");
+    }
+  }
+
+  if (!dry_run_) {
+    status = CreateMissingDirs();
+    if (!status.ok()) {
+      return WrapStatus(status, "Failed to create missing directories");
+    }
+  }
+
+  // Send indices of missing files to the client
+  status = SendFileIndices("missing", diff_.missing_files);
+  if (!status.ok()) {
+    return WrapStatus(status, "Failed to send indices of missing files");
+  }
+
+  if (!dry_run_) {
+    // Get file data of missing files from client.
+    status = HandleSendMissingFileData();
+    if (!status.ok()) {
+      return WrapStatus(status, "Failed to copy files");
+    }
+  }
+  // Send indices of changed files to the client.
+  status = SendFileIndices("changed", diff_.changed_files);
+  if (!status.ok()) {
+    return WrapStatus(status, "Failed to send indices of changed files");
+  }
+
+  if (!dry_run_) {
+    // Applies the rsync algorithm to update the changed files.
+    status = SyncChangedFiles();
+    if (!status.ok()) {
+      return WrapStatus(status, "Failed to sync files");
+    }
+  }
+
+  // Handle clean shutdown.
+  status = HandleShutdown();
+  if (!status.ok()) {
+    return WrapStatus(status, "Shutdown failed");
+  }
+
+  return absl::OkStatus();
+}
+
+absl::Status GgpRsyncServer::HandleSetOptions() {
+  LOG_INFO("Receiving options");
+
+  SetOptionsRequest request;
+  absl::Status status =
+      message_pump_->ReceiveMessage(PacketType::kSetOptions, &request);
+  if (!status.ok()) {
+    return WrapStatus(status, "Failed to receive SetOptionsRequest");
+  }
+
+  destination_ = request.destination();
+  delete_ = request.delete_();
+  recursive_ = request.recursive();
+  verbosity_ = request.verbosity();
+  whole_file_ = request.whole_file();
+  compress_ = request.compress();
+  checksum_ = request.checksum();
+  relative_ = request.relative();
+  dry_run_ = request.dry_run();
+  existing_ = request.existing();
+  copy_dest_ = request.copy_dest();
+
+  // (internal): Support \ instead of / in destination folders.
+  path::FixPathSeparators(&destination_);
+  path::EnsureEndsWithPathSeparator(&destination_);
+  if (!copy_dest_.empty()) {
+    path::FixPathSeparators(&copy_dest_);
+    path::EnsureEndsWithPathSeparator(&copy_dest_);
+  }
+
+  assert(path_filter_.IsEmpty());
+  for (int n = 0; n < request.filter_rules_size(); ++n) {
+    std::string fixed_pattern = request.filter_rules(n).pattern();
+    path::FixPathSeparators(&fixed_pattern);
+    path_filter_.AddRule(ToInternalType(request.filter_rules(n).type()),
+                         fixed_pattern);
+  }
+
+  Log::Instance()->SetLogLevel(Log::VerbosityToLogLevel(verbosity_));
+
+  return absl::OkStatus();
+}
+
+absl::Status GgpRsyncServer::FindFiles() {
+  Stopwatch stopwatch;
+  FileFinder finder;
+
+  LOG_INFO("Finding all files in destination folder '%s'", destination_);
+  absl::Status status =
+      finder.AddFiles(destination_, recursive_, &path_filter_);
+  if (!status.ok()) {
+    return WrapStatus(status, "Failed to search '%s'", destination_);
+  }
+
+  if (!copy_dest_.empty()) {
+    LOG_INFO("Finding all files in copy-dest folder '%s'", copy_dest_);
+    status = finder.AddFiles(copy_dest_, recursive_, &path_filter_);
+    if (!status.ok()) {
+      return WrapStatus(status, "Failed to search '%s'", copy_dest_);
+    }
+  }
+
+  finder.ReleaseFiles(&server_files_, &server_dirs_);
+
+  LOG_INFO("Found and set %u source files in %0.3f seconds",
+           server_files_.size(), stopwatch.ElapsedSeconds());
+  return absl::OkStatus();
+}
+
+absl::Status GgpRsyncServer::HandleSendAllFiles() {
+  std::string current_directory;
+
+  for (;;) {
+    AddFilesRequest request;
+    absl::Status status =
+        message_pump_->ReceiveMessage(PacketType::kAddFiles, &request);
+    if (!status.ok()) {
+      return WrapStatus(status, "Failed to receive AddFilesRequest");
+    }
+
+    // An empty request indicates that all files have been sent.
+    if (request.files_size() == 0 && request.dirs_size() == 0) {
+      return absl::OkStatus();
+    }
+
+    current_directory = request.directory();
+    path::FixPathSeparators(&current_directory);
+
+    // Add client files.
+    for (const AddFilesRequest::File& file : request.files()) {
+      uint32_t client_index = client_files_.size();
+      client_files_.emplace_back(path::Join(current_directory, file.filename()),
+                                 file.modified_time(), file.size(),
+                                 client_index, nullptr);
+    }
+    // Add client directories.
+    for (const std::string& dir : request.dirs()) {
+      uint32_t client_index = client_dirs_.size();
+      client_dirs_.emplace_back(path::Join(current_directory, dir),
+                                client_index, nullptr);
+    }
+  }
+}
+
+absl::Status GgpRsyncServer::DiffFiles() {
+  LOG_INFO("Diffing files");
+
+  // Be sure to move the data. It can grow quite large with millions of files.
+  // Special case for relative, non-recursive mode. This puts files with a
+  // relative directory into the "missing" bucket since the server-side search
+  // doesn't look into sub-folders. Double check that they are really missing.
+  const bool double_check_missing = relative_ && !recursive_;
+  diff_ =
+      file_diff::Generate(std::move(client_files_), std::move(server_files_),
+                          std::move(client_dirs_), std::move(server_dirs_),
+                          destination_, copy_dest_, double_check_missing);
+
+  // Take sync flags into account and generate the stats response.
+  SendFileStatsResponse response = file_diff::AdjustToFlagsAndGetStats(
+      existing_, checksum_, whole_file_, &diff_);
+
+  // Send stats.
+  absl::Status status =
+      message_pump_->SendMessage(PacketType::kSendFileStats, response);
+  if (!status.ok()) {
+    return WrapStatus(status, "Failed to send SendFileStatsResponse");
+  }
+
+  return absl::OkStatus();
+}
+
+absl::Status GgpRsyncServer::RemoveExtraneousFilesAndDirs() {
+  FileDeleterAndSender deleter(message_pump_.get());
+
+  // To guarantee that the folders are empty before they are removed, files are
+  // removed first.
+  for (const FileInfo& file : diff_.extraneous_files) {
+    absl::Status status = deleter.DeleteAndSendFileOrDir(
+        destination_, file.filepath, dry_run_, false);
+    if (!status.ok()) {
+      return WrapStatus(status, "Failed to delete file '%s' and send info",
+                        file.filepath);
+    }
+  }
+
+  // To guarantee that the subfolders are removed first.
+  std::sort(diff_.extraneous_dirs.begin(), diff_.extraneous_dirs.end(),
+            [](const DirInfo& dir1, const DirInfo& dir2) {
+              return dir1.filepath > dir2.filepath;
+            });
+  for (const DirInfo& dir : diff_.extraneous_dirs) {
+    absl::Status status = deleter.DeleteAndSendFileOrDir(
+        destination_, dir.filepath, dry_run_, true);
+    if (!status.ok()) {
+      return WrapStatus(status, "Failed to delete directory '%s' and send info",
+                        dir.filepath);
+    }
+  }
+
+  // Send remaining files to the client.
+  absl::Status status = deleter.Flush();
+  if (!status.ok()) {
+    return WrapStatus(
+        status,
+        "Failed to send info of remaining deleted files and directories");
+  }
+
+  return absl::OkStatus();
+}
+
+absl::Status GgpRsyncServer::CreateMissingDirs() {
+  for (const DirInfo& dir : diff_.missing_dirs) {
+    // Make directory.
+    std::string path = path::Join(destination_, dir.filepath);
+    std::error_code error_code;
+    // A file with the same name already exists.
+    if (path::Exists(path)) {
+      assert(!diff_.extraneous_files.empty());
+      absl::Status status = path::RemoveFile(path);
+      if (!status.ok()) {
+        return WrapStatus(
+            status, "Failed to remove file '%s' before creating folder '%s'",
+            path, path);
+      }
+    }
+    absl::Status status = path::CreateDirRec(path);
+    if (!status.ok()) {
+      return WrapStatus(status, "Failed to create directory %s", path);
+    }
+  }
+  return absl::OkStatus();
+}
+
+template <typename T>
+absl::Status GgpRsyncServer::SendFileIndices(const char* file_type,
+                                             const std::vector<T>& files) {
+  LOG_INFO("Sending indices of missing files to client");
+  constexpr char error_fmt[] = "Failed to send indices of %s files.";
+
+  AddFileIndicesResponse response;
+  absl::Status status;
+  for (const T& file : files) {
+    response.add_client_indices(file.client_index);
+
+    constexpr int kMaxBatchSize = 4000;
+    if (response.client_indices_size() >= kMaxBatchSize) {
+      status =
+          message_pump_->SendMessage(PacketType::kAddFileIndices, response);
+      response.clear_client_indices();
+    }
+
+    if (!status.ok()) {
+      return WrapStatus(status, error_fmt, file_type);
+    }
+  }
+
+  // Send the rest.
+  if (response.client_indices_size() > 0) {
+    status = message_pump_->SendMessage(PacketType::kAddFileIndices, response);
+    if (!status.ok()) {
+      return WrapStatus(status, error_fmt, file_type);
+    }
+
+    response.clear_client_indices();
+  }
+
+  // Send an empty response to indicate that we're done.
+  status = message_pump_->SendMessage(PacketType::kAddFileIndices, response);
+  if (!status.ok()) {
+    return WrapStatus(status, error_fmt, file_type);
+  }
+
+  return absl::OkStatus();
+}
+
+absl::Status GgpRsyncServer::HandleSendMissingFileData() {
+  if (diff_.missing_files.empty()) {
+    return absl::OkStatus();
+  }
+
+  LOG_INFO("Receiving missing files");
+
+  // Expect start of compression. The server socket will actually handle
+  // compression transparently, there's nothing we have to do here.
+  if (compress_) {
+    ToggleCompressionRequest request;
+    absl::Status status =
+        message_pump_->ReceiveMessage(PacketType::kToggleCompression, &request);
+    if (!status.ok()) {
+      return WrapStatus(status, "Failed to receive ToggleCompressionRequest");
+    }
+  }
+
+  for (uint32_t server_index = 0; server_index < diff_.missing_files.size();
+       server_index++) {
+    const FileInfo& file = diff_.missing_files[server_index];
+    std::string filepath = path::Join(destination_, file.filepath);
+    LOG_INFO("%s", filepath.c_str());
+
+    SendMissingFileDataRequest request;
+    absl::Status status = message_pump_->ReceiveMessage(
+        PacketType::kSendMissingFileData, &request);
+    if (!status.ok()) {
+      return WrapStatus(status, "Failed to receive SendMissingFileDataRequest");
+    }
+
+    // Verify if we got the right index.
+    if (request.server_index() != server_index) {
+      return MakeStatus("Received wrong server index %u. Expected %u.",
+                        request.server_index(), server_index);
+    }
+
+    // Verify that there is no directory existing with the same name.
+    if (path::Exists(filepath) && path::DirExists(filepath)) {
+      assert(!diff_.extraneous_dirs.empty());
+      absl::Status status = path::RemoveFile(filepath);
+      if (!status.ok()) {
+        return WrapStatus(
+            status, "Failed to remove folder '%s' before creating file '%s'",
+            filepath, filepath);
+      }
+    }
+
+    // Make directory.
+    std::string dir = path::DirName(filepath);
+    std::error_code error_code;
+    status = path::CreateDirRec(dir);
+    if (!status.ok()) {
+      return MakeStatus("Failed to create directory %s: %s", dir,
+                        error_code.message());
+    }
+
+    // Receive file data.
+    Buffer buffer;
+    bool is_executable = false;
+    bool first_chunk = true;
+    auto handler = [message_pump = message_pump_.get(), &buffer, &is_executable,
+                    &first_chunk](const void** data, size_t* size) {
+      absl::Status status = message_pump->ReceiveRawData(&buffer);
+      if (!status.ok()) {
+        return status;
+      }
+
+      // size 0 indicates EOF.
+      *data = buffer.size() > 0 ? buffer.data() : nullptr;
+      *size = buffer.size();
+
+      // Detect executables.
+      if (first_chunk && buffer.size() > 0) {
+        first_chunk = false;
+        is_executable = Util::IsExecutable(buffer.data(), buffer.size());
+      }
+
+      return absl::OkStatus();
+    };
+
+    absl::StatusOr<FILE*> fp = path::OpenFile(filepath, "wb");
+    if (!fp.ok()) {
+      return fp.status();
+    }
+
+    status = path::StreamWriteFileContents(*fp, handler);
+    fclose(*fp);
+    if (!status.ok()) {
+      return WrapStatus(status, "Failed to write file %s", filepath);
+    }
+
+    // Set file write time.
+    status = path::SetFileTime(filepath, file.modified_time);
+    if (!status.ok()) {
+      return WrapStatus(status, "Failed to set file mod time for %s", filepath);
+    }
+
+    // Set executable bit, but just print warnings as it's not critical.
+    if (is_executable) {
+      path::Stats stats;
+      status = path::GetStats(filepath, &stats);
+      if (status.ok()) {
+        status = path::ChangeMode(filepath, stats.mode | kExecutableBits);
+      }
+      if (!status.ok()) {
+        LOG_WARNING("Failed to set executable bit on '%s': %s", filepath,
+                    status.ToString());
+      }
+    }
+  }
+
+  // Notify client that it can resume sending (uncompressed!) messages.
+  if (compress_) {
+    ToggleCompressionResponse response;
+    absl::Status status =
+        message_pump_->SendMessage(PacketType::kToggleCompression, response);
+    if (!status.ok()) {
+      return WrapStatus(status, "Failed to send ToggleCompressionResponse");
+    }
+  }
+
+  return absl::OkStatus();
+}
+
+absl::Status GgpRsyncServer::SyncChangedFiles() {
+  if (diff_.changed_files.empty()) {
+    return absl::OkStatus();
+  }
+
+  LOG_INFO("Synching changed files");
+
+  // Expect start of compression. The server socket will actually handle
+  // compression transparently, there's nothing we have to do here.
+  if (compress_) {
+    ToggleCompressionRequest request;
+    absl::Status status =
+        message_pump_->ReceiveMessage(PacketType::kToggleCompression, &request);
+    if (!status.ok()) {
+      return WrapStatus(status, "Failed to receive ToggleCompressionRequest");
+    }
+  }
+
+  CdcInterface cdc(message_pump_.get());
+
+  // Pipeline sending signatures and patching files:
+  // MAIN THREAD:   Send signatures to client.
+  //                Only sends to the socket.
+  // WORKER THREAD: Receive diffs from client and patch file.
+  //                Only reads from the socket.
+  Threadpool pool(1);
+
+  for (uint32_t server_index = 0; server_index < diff_.changed_files.size();
+       server_index++) {
+    const ChangedFileInfo& file = diff_.changed_files[server_index];
+    std::string base_filepath =
+        path::Join(file.base_dir ? file.base_dir : destination_, file.filepath);
+    std::string target_filepath = path::Join(destination_, file.filepath);
+    LOG_INFO("%s -> %s", base_filepath, target_filepath);
+
+    SendSignatureResponse response;
+    response.set_client_index(file.client_index);
+    response.set_server_file_size(file.server_size);
+    absl::Status status =
+        message_pump_->SendMessage(PacketType::kAddSignatures, response);
+    if (!status.ok()) {
+      return WrapStatus(status, "Failed to send SendSignatureResponse");
+    }
+
+    // Create and send signature.
+    status = cdc.CreateAndSendSignature(base_filepath);
+    if (!status.ok()) {
+      return status;
+    }
+
+    // Queue patching task.
+    pool.QueueTask(std::make_unique<PatchTask>(base_filepath, target_filepath,
+                                               file, &cdc));
+
+    // Wait for the last file to finish.
+    if (server_index + 1 == diff_.changed_files.size()) {
+      pool.Wait();
+    }
+
+    // Check the results of completed tasks.
+    std::unique_ptr<Task> task = pool.TryGetCompletedTask();
+    while (task) {
+      PatchTask* patch_task = static_cast<PatchTask*>(task.get());
+      const std::string& task_path = patch_task->File().filepath;
+      if (!patch_task->Status().ok()) {
+        return WrapStatus(patch_task->Status(), "Failed to patch file '%s'",
+                          task_path);
+      }
+      LOG_INFO("Finished patching file %s", task_path.c_str());
+      task = pool.TryGetCompletedTask();
+    }
+  }
+
+  // Notify client that it can resume sending (uncompressed!) messages.
+  if (compress_) {
+    ToggleCompressionResponse response;
+    absl::Status status =
+        message_pump_->SendMessage(PacketType::kToggleCompression, response);
+    if (!status.ok()) {
+      return WrapStatus(status, "Failed to send ToggleCompressionResponse");
+    }
+  }
+
+  LOG_INFO("Successfully synced %u files", diff_.changed_files.size());
+
+  return absl::OkStatus();
+}
+
+absl::Status GgpRsyncServer::HandleShutdown() {
+  ShutdownRequest request;
+  absl::Status status =
+      message_pump_->ReceiveMessage(PacketType::kShutdown, &request);
+  if (!status.ok()) {
+    return WrapStatus(status, "Failed to receive ShutdownRequest");
+  }
+
+  ShutdownResponse response;
+  status = message_pump_->SendMessage(PacketType::kShutdown, response);
+  if (!status.ok()) {
+    return WrapStatus(status, "Failed to send ShutdownResponse");
+  }
+
+  return absl::OkStatus();
+}
+
+void GgpRsyncServer::Thread_OnPackageReceived(PacketType type) {
+  if (type != PacketType::kToggleCompression) {
+    return;
+  }
+
+  // Turn on decompression.
+  message_pump_->RedirectInput(std::make_unique<UnzstdStream>(socket_.get()));
+}
+
+}  // namespace cdc_ft

cdc_rsync_server/cdc_rsync_server.h

@@ -0,0 +1,121 @@
+/*
+ * Copyright 2022 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef CDC_RSYNC_SERVER_CDC_RSYNC_SERVER_H_
+#define CDC_RSYNC_SERVER_CDC_RSYNC_SERVER_H_
+
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "absl/status/status.h"
+#include "cdc_rsync/base/message_pump.h"
+#include "cdc_rsync_server/file_diff_generator.h"
+#include "cdc_rsync_server/file_info.h"
+#include "common/gamelet_component.h"
+#include "common/path_filter.h"
+
+namespace cdc_ft {
+
+class MessagePump;
+class ServerSocket;
+
+class GgpRsyncServer {
+ public:
+  GgpRsyncServer();
+  ~GgpRsyncServer();
+
+  // Checks that the gamelet components (cdc_rsync_server binary etc.) are
+  // up-to-date by checking their sizes and timestamps.
+  bool CheckComponents(const std::vector<GameletComponent>& components);
+
+  // Listens to |port|, accepts a connection from the client and runs the rsync
+  // procedure.
+  absl::Status Run(int port);
+
+  // Returns the verbosity sent from the client. 0 by default.
+  int GetVerbosity() const { return verbosity_; }
+
+ private:
+  // Runs the rsync procedure.
+  absl::Status Sync();
+
+  // Receives options from the client.
+  absl::Status HandleSetOptions();
+
+  // Finds all server-side files in the |destination_| folder.
+  absl::Status FindFiles();
+
+  // Receives all client-side files.
+  absl::Status HandleSendAllFiles();
+
+  // Diffs client- and server-side files.
+  absl::Status DiffFiles();
+
+  // Deletes files and directories present on the server, but not on the client.
+  absl::Status RemoveExtraneousFilesAndDirs();
+
+  // Creates missing directories.
+  absl::Status CreateMissingDirs();
+
+  // Sends file indices to the client. Used for missing and changed files.
+  template <typename T>
+  absl::Status SendFileIndices(const char* file_type,
+                               const std::vector<T>& files);
+
+  // Receives missing files from the client.
+  absl::Status HandleSendMissingFileData();
+
+  // Core rsync algorithm. Sends signatures of changed files to the client,
+  // receives diffs and applies them.
+  absl::Status SyncChangedFiles();
+
+  // Waits for the shutdown message and send an ack.
+  absl::Status HandleShutdown();
+
+  // Called on |message_pump_|'s receiver thread whenever a package is received.
+  // Used to toggle decompression.
+  void Thread_OnPackageReceived(PacketType type);
+
+  std::unique_ptr<ServerSocket> socket_;
+  std::unique_ptr<MessagePump> message_pump_;
+
+  std::string destination_;
+
+  // Options.
+  bool delete_ = false;
+  bool recursive_ = false;
+  int verbosity_ = 0;
+  bool whole_file_ = false;
+  bool compress_ = false;
+  bool checksum_ = false;
+  bool relative_ = false;
+  bool dry_run_ = false;
+  bool existing_ = false;
+  std::string copy_dest_;
+
+  PathFilter path_filter_;
+
+  std::vector<FileInfo> client_files_;
+  std::vector<DirInfo> client_dirs_;
+  std::vector<FileInfo> server_files_;
+  std::vector<DirInfo> server_dirs_;
+  file_diff::Result diff_;
+};
+
+}  // namespace cdc_ft
+
+#endif  // CDC_RSYNC_SERVER_CDC_RSYNC_SERVER_H_

cdc_rsync_server/cdc_rsync_server.vcxproj

@@ -0,0 +1,59 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project DefaultTargets="Build" ToolsVersion="14.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <ItemGroup Label="ProjectConfigurations">
+    <ProjectConfiguration Include="Debug|GGP">
+      <Configuration>Debug</Configuration>
+      <Platform>GGP</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Release|GGP">
+      <Configuration>Release</Configuration>
+      <Platform>GGP</Platform>
+    </ProjectConfiguration>
+  </ItemGroup>
+  <PropertyGroup Label="Globals">
+    <ProjectGuid>{4ece65e0-d950-4b96-8ad5-0313261b8c8d}</ProjectGuid>
+    <RootNamespace>cdc_rsync_server</RootNamespace>
+  </PropertyGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|GGP'" Label="Configuration">
+    <ConfigurationType>Makefile</ConfigurationType>
+    <UseDebugLibraries>true</UseDebugLibraries>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|GGP'" Label="Configuration">
+    <ConfigurationType>Makefile</ConfigurationType>
+    <UseDebugLibraries>false</UseDebugLibraries>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|GGP'">
+    <OutDir>$(SolutionDir)bazel-out\k8-dbg\bin\cdc_rsync_server\</OutDir>
+    <AdditionalOptions>/std:c++17</AdditionalOptions>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|GGP'">
+    <OutDir>$(SolutionDir)bazel-out\k8-opt\bin\cdc_rsync_server\</OutDir>
+    <AdditionalOptions>/std:c++17</AdditionalOptions>
+  </PropertyGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
+  <ImportGroup Label="ExtensionSettings">
+  </ImportGroup>
+  <ImportGroup Label="Shared">
+    <Import Project="..\all_files.vcxitems" Label="Shared" />
+  </ImportGroup>
+  <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Debug|GGP'">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Release|GGP'">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <PropertyGroup Label="UserMacros">
+  </PropertyGroup>
+  <!-- Bazel setup -->
+  <PropertyGroup>
+    <BazelTargets>//cdc_rsync_server:cdc_rsync_server</BazelTargets>
+    <BazelOutputFile>cdc_rsync_server</BazelOutputFile>
+    <BazelIncludePaths>..\;..\third_party\absl;..\third_party\blake3\c;..\bazel-stadia-file-transfer\external\com_github_zstd\lib;..\third_party\googletest\googletest\include;..\third_party\protobuf\src</BazelIncludePaths>
+    <BazelSourcePathPrefix>..\/</BazelSourcePathPrefix>
+  </PropertyGroup>
+  <Import Project="..\NMakeBazelProject.targets" />
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
+  <ImportGroup Label="ExtensionTargets">
+  </ImportGroup>
+</Project>

cdc_rsync_server/cdc_rsync_server.vcxproj.filters

@@ -0,0 +1,2 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003" />

cdc_rsync_server/file_deleter_and_sender.cc

@@ -0,0 +1,111 @@
+// Copyright 2022 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "cdc_rsync_server/file_deleter_and_sender.h"
+
+#include "cdc_rsync/base/message_pump.h"
+#include "common/log.h"
+#include "common/path.h"
+#include "common/status.h"
+
+namespace cdc_ft {
+
+FileDeleterAndSender::FileDeleterAndSender(MessagePump* message_pump,
+                                           size_t request_size_threshold)
+    : message_pump_(message_pump),
+      request_size_threshold_(request_size_threshold) {
+  assert(message_pump_);
+}
+
+FileDeleterAndSender::~FileDeleterAndSender() = default;
+
+absl::Status FileDeleterAndSender::DeleteAndSendFileOrDir(
+    const std::string& base_dir, const std::string& relative_path, bool dry_run,
+    bool is_directory) {
+  std::string filepath = path::Join(base_dir, relative_path);
+  if (!dry_run) {
+    LOG_INFO("Removing %s", filepath.c_str());
+    absl::Status status = path::RemoveFile(filepath);
+    if (!status.ok()) {
+      return WrapStatus(status, "Failed to remove '%s'", filepath);
+    }
+  }
+
+  std::string relative_dir = path::DirName(relative_path);
+  if (!relative_dir.empty()) path::EnsureEndsWithPathSeparator(&relative_dir);
+  if (response_.directory() != relative_dir) {
+    // Flush files in previous directory.
+    absl::Status status = SendFilesAndDirs();
+    if (!status.ok()) {
+      return WrapStatus(
+          status, "Failed to send deleted files and directories to client");
+    }
+
+    // Set new directory.
+    response_.set_directory(relative_dir);
+    response_size_ = response_.directory().length();
+  }
+
+  std::string filename = path::BaseName(relative_path);
+  if (is_directory) {
+    *response_.add_dirs() = filename;
+  } else {
+    *response_.add_files() = filename;
+  }
+  response_size_ += filename.size();
+
+  if (response_size_ >= request_size_threshold_) {
+    absl::Status status = SendFilesAndDirs();
+    if (!status.ok()) {
+      return WrapStatus(
+          status, "Failed to send deleted files and directories to client");
+    }
+  }
+
+  return absl::OkStatus();
+}
+
+absl::Status FileDeleterAndSender::Flush() {
+  absl::Status status = SendFilesAndDirs();
+  if (!status.ok()) {
+    return WrapStatus(status,
+                      "Failed to send deleted files and directories to client");
+  }
+
+  // Send an empty batch as EOF indicator.
+  assert(response_.files_size() == 0 && response_.dirs_size() == 0);
+  status = message_pump_->SendMessage(PacketType::kAddDeletedFiles, response_);
+  if (!status.ok()) {
+    return WrapStatus(status, "Failed to send EOF indicator");
+  }
+
+  return absl::OkStatus();
+}
+
+absl::Status FileDeleterAndSender::SendFilesAndDirs() {
+  if (response_.files_size() == 0 && response_.dirs_size() == 0) {
+    return absl::OkStatus();
+  }
+
+  absl::Status status =
+      message_pump_->SendMessage(PacketType::kAddDeletedFiles, response_);
+  if (!status.ok()) {
+    return WrapStatus(status, "Failed to send AddDeletedFilesResponse");
+  }
+  response_.Clear();
+  response_size_ = response_.directory().length();
+  return absl::OkStatus();
+}
+
+}  // namespace cdc_ft

cdc_rsync_server/file_deleter_and_sender.h

@@ -0,0 +1,65 @@
+/*
+ * Copyright 2022 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef CDC_RSYNC_SERVER_FILE_DELETER_AND_SENDER_H_
+#define CDC_RSYNC_SERVER_FILE_DELETER_AND_SENDER_H_
+
+#include <string>
+
+#include "absl/status/status.h"
+#include "cdc_rsync/protos/messages.pb.h"
+
+namespace cdc_ft {
+
+class MessagePump;
+
+// Deletes files and sends info about deleted files to the client.
+class FileDeleterAndSender {
+ public:
+  // Send AddDeletedFileResponse in packets of roughly 10k max by default.
+  static constexpr size_t kDefaultResponseSizeThreshold = 10000;
+
+  FileDeleterAndSender(
+      MessagePump* message_pump,
+      size_t response_size_threshold = kDefaultResponseSizeThreshold);
+  ~FileDeleterAndSender();
+
+  // Deletes |base_dir| + |relative_path| and send |relative_path| the client.
+  // Deletion happens for either a directory or a file and only in a non dry-run
+  // mode.
+  absl::Status DeleteAndSendFileOrDir(const std::string& base_dir,
+                                      const std::string& relative_path,
+                                      bool dry_run, bool is_directory);
+
+  // Sends the remaining file and directory batch to the client, followed by an
+  // EOF indicator. Should be called once all files and directories have been
+  // deleted.
+  absl::Status Flush();
+
+ private:
+  // Sends the current batch to the client.
+  absl::Status SendFilesAndDirs();
+
+  MessagePump* const message_pump_;
+  const size_t request_size_threshold_;
+
+  AddDeletedFilesResponse response_;
+  size_t response_size_ = 0;
+};
+
+}  // namespace cdc_ft
+
+#endif  // CDC_RSYNC_SERVER_FILE_DELETER_AND_SENDER_H_

cdc_rsync_server/file_deleter_and_sender_test.cc

@@ -0,0 +1,263 @@
+// Copyright 2022 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "cdc_rsync_server/file_deleter_and_sender.h"
+
+#include "cdc_rsync/base/fake_socket.h"
+#include "cdc_rsync/base/message_pump.h"
+#include "common/log.h"
+#include "common/path.h"
+#include "common/status_test_macros.h"
+#include "gtest/gtest.h"
+
+constexpr bool kFile = false;
+constexpr bool kDir = true;
+
+constexpr bool kDryRun = true;
+constexpr bool kNoDryRun = false;
+
+namespace cdc_ft {
+namespace {
+
+// Note: FileDiffGenerator is a server-only class and only runs on GGP, but the
+// code is independent of the platform, so we can test it from Windows.
+class FileDeleterAndSenderTest : public ::testing::Test {
+  void SetUp() override {
+    Log::Initialize(std::make_unique<ConsoleLog>(LogLevel::kInfo));
+    message_pump_.StartMessagePump();
+    tmp_dir_ = path::GetTempDir();
+    path::EnsureDoesNotEndWithPathSeparator(&tmp_dir_);
+  }
+
+  void TearDown() override {
+    // Make sure there are no more AddDeletedFilesResponse.
+    ShutdownRequest shutdown;
+    EXPECT_OK(message_pump_.SendMessage(PacketType::kShutdown, shutdown));
+    EXPECT_OK(message_pump_.ReceiveMessage(PacketType::kShutdown, &shutdown));
+
+    socket_.ShutdownSendingEnd();
+    message_pump_.StopMessagePump();
+    Log::Shutdown();
+  }
+
+ protected:
+  // Creates a temp file in %TMP% with given |relative_path|.
+  std::string CreateTempFile(std::string relative_path) {
+    std::string full_path = path::Join(tmp_dir_, relative_path);
+
+    std::string dir = path::DirName(full_path);
+    EXPECT_OK(path::CreateDirRec(dir));
+    EXPECT_OK(path::WriteFile(full_path, ""));
+    EXPECT_TRUE(path::Exists(full_path));
+    return full_path;
+  }
+
+  // Creates a bunch of temp files in %TMP% with given |relative_paths|.
+  std::vector<std::string> CreateTempFiles(
+      std::vector<std::string> relative_paths) {
+    std::vector<std::string> full_paths;
+    for (const std::string& relative_path : relative_paths) {
+      full_paths.push_back(CreateTempFile(relative_path));
+    }
+    return full_paths;
+  }
+
+  // Creates a temp directory in %TMP% with given |relative_path|.
+  std::string CreateTempDir(std::string relative_path) {
+    std::string full_path = path::Join(tmp_dir_, relative_path);
+
+    EXPECT_OK(path::CreateDirRec(full_path));
+    EXPECT_TRUE(path::Exists(full_path));
+    return full_path;
+  }
+
+  // Creates a bunch of temp directories in %TMP% with given |relative_paths|.
+  std::vector<std::string> CreateTempDirs(
+      std::vector<std::string> relative_paths) {
+    std::vector<std::string> full_paths;
+    for (const std::string& relative_path : relative_paths) {
+      full_paths.push_back(CreateTempDir(relative_path));
+    }
+    return full_paths;
+  }
+
+  // Expects an AddDeletedFilesResponse with no files as EOF indicator.
+  void ExpectEofMarker() {
+    // Verify that there is only the empty "EOF" indicator message.
+    AddDeletedFilesResponse response;
+    EXPECT_OK(
+        message_pump_.ReceiveMessage(PacketType::kAddDeletedFiles, &response));
+    EXPECT_EQ(response.files_size(), 0);
+  }
+
+  FakeSocket socket_;
+  MessagePump message_pump_{&socket_, MessagePump::PacketReceivedDelegate()};
+
+  std::string tmp_dir_;
+};
+
+TEST_F(FileDeleterAndSenderTest, NoFiles) {
+  // Delete no files, no dirs.
+  FileDeleterAndSender deleter(&message_pump_);
+  EXPECT_OK(deleter.Flush());
+
+  ExpectEofMarker();
+}
+
+TEST_F(FileDeleterAndSenderTest, FilesDeletedAndSent) {
+  // Create temp files.
+  std::vector<std::string> full_paths = CreateTempFiles(
+      {"__fdas_unittest_1.txt", "__fdas_unittest_2.txt",
+       path::ToNative("__fdas_unittest_dir/__fdas_unittest_3.txt")});
+
+  // Delete files.
+  FileDeleterAndSender deleter(&message_pump_);
+  for (const std::string& file : full_paths) {
+    EXPECT_OK(deleter.DeleteAndSendFileOrDir(
+        tmp_dir_, file.substr(tmp_dir_.size()), kNoDryRun, kFile));
+  }
+  EXPECT_OK(deleter.Flush());
+
+  // Did the files get deleted?
+  for (const std::string& file : full_paths) {
+    EXPECT_FALSE(path::Exists(file));
+  }
+
+  // Verify that the data sent to the socket matches.
+  AddDeletedFilesResponse response;
+  EXPECT_OK(
+      message_pump_.ReceiveMessage(PacketType::kAddDeletedFiles, &response));
+  EXPECT_EQ(response.directory(), path::ToNative("/"));
+  ASSERT_EQ(response.files_size(), 2);
+  ASSERT_EQ(response.dirs_size(), 0);
+  EXPECT_EQ(response.files(0), "__fdas_unittest_1.txt");
+  EXPECT_EQ(response.files(1), "__fdas_unittest_2.txt");
+
+  EXPECT_OK(
+      message_pump_.ReceiveMessage(PacketType::kAddDeletedFiles, &response));
+  ASSERT_EQ(response.dirs_size(), 0);
+  EXPECT_EQ(response.directory(), path::ToNative("/__fdas_unittest_dir/"));
+  ASSERT_EQ(response.files_size(), 1);
+  EXPECT_EQ(response.files(0), "__fdas_unittest_3.txt");
+
+  ExpectEofMarker();
+}
+
+TEST_F(FileDeleterAndSenderTest, DirsDeletedAndSent) {
+  // Create temp dirs.
+  std::vector<std::string> full_paths = CreateTempDirs(
+      {"__fdas_unittest_dir",
+       path::ToNative("__fdas_unittest_dir/__fdas_unittest_1"),
+       path::ToNative("__fdas_unittest_dir/__fdas_unittest_2"),
+       path::ToNative(
+           "__fdas_unittest_dir/__fdas_unittest_1/__fdas_unittest_1_1")});
+
+  // Delete files.
+  FileDeleterAndSender deleter(&message_pump_);
+  for (size_t idx = full_paths.size(); idx > 0; --idx) {
+    EXPECT_OK(deleter.DeleteAndSendFileOrDir(
+        tmp_dir_, full_paths[idx - 1].substr(tmp_dir_.size() + 1), kNoDryRun,
+        kDir));
+  }
+  EXPECT_OK(deleter.Flush());
+
+  // Did the dirs get deleted?
+  for (const std::string& dir : full_paths) {
+    EXPECT_FALSE(path::Exists(dir));
+  }
+
+  // Verify that the data sent to the socket matches.
+  AddDeletedFilesResponse response;
+  EXPECT_OK(
+      message_pump_.ReceiveMessage(PacketType::kAddDeletedFiles, &response));
+  EXPECT_EQ(response.directory(),
+            path::ToNative("__fdas_unittest_dir/__fdas_unittest_1/"));
+  ASSERT_EQ(response.files_size(), 0);
+  ASSERT_EQ(response.dirs_size(), 1);
+  EXPECT_EQ(response.dirs(0), "__fdas_unittest_1_1");
+
+  EXPECT_OK(
+      message_pump_.ReceiveMessage(PacketType::kAddDeletedFiles, &response));
+  EXPECT_EQ(response.directory(), path::ToNative("__fdas_unittest_dir/"));
+  ASSERT_EQ(response.files_size(), 0);
+  ASSERT_EQ(response.dirs_size(), 2);
+  EXPECT_EQ(response.dirs(0), "__fdas_unittest_2");
+  EXPECT_EQ(response.dirs(1), "__fdas_unittest_1");
+
+  EXPECT_OK(
+      message_pump_.ReceiveMessage(PacketType::kAddDeletedFiles, &response));
+  EXPECT_EQ(response.directory(), "");
+  ASSERT_EQ(response.files_size(), 0);
+  ASSERT_EQ(response.dirs_size(), 1);
+  EXPECT_EQ(response.dirs(0), "__fdas_unittest_dir");
+
+  ExpectEofMarker();
+}
+
+TEST_F(FileDeleterAndSenderTest, FilesDeletedAndSentDryRun) {
+  // Create a temp file.
+  std::string file_to_remove = CreateTempFile("__fdas_unittest_1.txt");
+
+  // "Delete" the file in dry-run mode: the file should not be deleted.
+  // It should be just sent to the socket.
+  FileDeleterAndSender deleter(&message_pump_);
+  EXPECT_OK(deleter.DeleteAndSendFileOrDir(
+      tmp_dir_, file_to_remove.substr(tmp_dir_.size()), kDryRun, kFile));
+  EXPECT_OK(deleter.Flush());
+
+  EXPECT_TRUE(path::Exists(file_to_remove));
+
+  // Verify that the data sent to the socket matches.
+  AddDeletedFilesResponse response;
+  EXPECT_OK(
+      message_pump_.ReceiveMessage(PacketType::kAddDeletedFiles, &response));
+  EXPECT_EQ(response.directory(), path::ToNative("/"));
+  ASSERT_EQ(response.files_size(), 1);
+  EXPECT_EQ(response.files(0), "__fdas_unittest_1.txt");
+
+  ExpectEofMarker();
+}
+
+TEST_F(FileDeleterAndSenderTest, MessageSplitByMaxSize) {
+  // Create temp files.
+  std::vector<std::string> full_paths =
+      CreateTempFiles({"__fdas_unittest_1.txt", "__fdas_unittest_2.txt"});
+
+  // Delete files. The size is picked so that the message gets split.
+  FileDeleterAndSender deleter(&message_pump_, /*max_request_byte_size=*/20);
+  for (const std::string& file : full_paths) {
+    EXPECT_OK(deleter.DeleteAndSendFileOrDir(
+        tmp_dir_, file.substr(tmp_dir_.size()), kNoDryRun, kFile));
+  }
+  EXPECT_OK(deleter.Flush());
+
+  // Verify that the data sent to the socket matches.
+  AddDeletedFilesResponse response;
+  EXPECT_OK(
+      message_pump_.ReceiveMessage(PacketType::kAddDeletedFiles, &response));
+  EXPECT_EQ(response.directory(), path::ToNative("/"));
+  ASSERT_EQ(response.files_size(), 1);
+  EXPECT_EQ(response.files(0), "__fdas_unittest_1.txt");
+
+  EXPECT_OK(
+      message_pump_.ReceiveMessage(PacketType::kAddDeletedFiles, &response));
+  EXPECT_EQ(response.directory(), path::ToNative("/"));
+  ASSERT_EQ(response.files_size(), 1);
+  EXPECT_EQ(response.files(0), "__fdas_unittest_2.txt");
+
+  ExpectEofMarker();
+}
+
+}  // namespace
+}  // namespace cdc_ft

cdc_rsync_server/file_diff_generator.cc

@@ -0,0 +1,287 @@
+// Copyright 2022 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "cdc_rsync_server/file_diff_generator.h"
+
+#include <algorithm>
+
+#include "common/log.h"
+#include "common/path.h"
+#include "common/util.h"
+
+namespace cdc_ft {
+namespace file_diff {
+
+namespace {
+
+struct FilePathComparer {
+  bool operator()(const FileInfo& a, const FileInfo& b) {
+    return a.filepath < b.filepath;
+  }
+};
+
+struct FilePathEquals {
+  bool operator()(const FileInfo& a, const FileInfo& b) {
+    return a.filepath == b.filepath;
+  }
+};
+
+struct DirPathComparer {
+  bool operator()(const DirInfo& a, const DirInfo& b) {
+    return a.filepath < b.filepath;
+  }
+};
+
+struct DirPathEquals {
+  bool operator()(const DirInfo& a, const DirInfo& b) {
+    return a.filepath == b.filepath;
+  }
+};
+
+bool FindFile(const std::string& base_dir, const std::string& relative_path,
+              FileInfo* file) {
+  path::Stats stats;
+  if (!path::GetStats(path::Join(base_dir, relative_path), &stats).ok()) {
+    return false;
+  }
+
+  *file = FileInfo(relative_path, stats.modified_time, stats.size,
+                   FileInfo::kInvalidIndex, base_dir.c_str());
+  return true;
+}
+
+// Returns true if |client_file| and |server_file| are considered a match based
+// on filesize and timestamp. An exception is if the |server_file| is in the
+// |copy_dest| directory (e.g. the package dir). In that case, the file should
+// be considered as changed (the sync algo will copy the file over).
+bool FilesMatch(const FileInfo& client_file, const FileInfo& server_file,
+                const std::string& copy_dest) {
+  return client_file.size == server_file.size &&
+         client_file.modified_time == server_file.modified_time &&
+         (copy_dest.empty() || server_file.base_dir != copy_dest.c_str());
+}
+
+}  // namespace
+
+Result Generate(std::vector<FileInfo>&& client_files,
+                std::vector<FileInfo>&& server_files,
+                std::vector<DirInfo>&& client_dirs,
+                std::vector<DirInfo>&& server_dirs, const std::string& base_dir,
+                const std::string& copy_dest, bool double_check_missing) {
+  // Sort both arrays by filepath.
+  std::sort(client_files.begin(), client_files.end(), FilePathComparer());
+  std::sort(server_files.begin(), server_files.end(), FilePathComparer());
+  std::sort(client_dirs.begin(), client_dirs.end(), DirPathComparer());
+  std::sort(server_dirs.begin(), server_dirs.end(), DirPathComparer());
+
+  // De-dupe client files, just in case. This might happen if someone calls
+  // cdc_rsync with overlapping sources, e.g. assets/* and assets/textures/*.
+  client_files.erase(
+      std::unique(client_files.begin(), client_files.end(), FilePathEquals()),
+      client_files.end());
+
+  client_dirs.erase(
+      std::unique(client_dirs.begin(), client_dirs.end(), DirPathEquals()),
+      client_dirs.end());
+
+  // Compare the arrays, sorting the files into the right buckets.
+  std::vector<FileInfo>::iterator client_iter = client_files.begin();
+  std::vector<FileInfo>::iterator server_iter = server_files.begin();
+
+  Result diff;
+
+  while (client_iter != client_files.end() ||
+         server_iter != server_files.end()) {
+    const int order =
+        client_iter == client_files.end() ? 1  // Extraneous.
+        : server_iter == server_files.end()
+            ? -1  // Missing.
+            : client_iter->filepath.compare(server_iter->filepath);
+
+    if (order < 0) {
+      // File is missing, but first double check if it's really missing if
+      // |double_check_missing| is true.
+      FileInfo server_file(std::string(), 0, 0, FileInfo::kInvalidIndex,
+                           nullptr);
+      bool found = false;
+      if (double_check_missing) {
+        found = FindFile(base_dir, client_iter->filepath, &server_file);
+        if (!found && !copy_dest.empty()) {
+          found = FindFile(copy_dest, client_iter->filepath, &server_file);
+        }
+      }
+
+      if (!found) {
+        diff.missing_files.push_back(std::move(*client_iter));
+      } else if (FilesMatch(*client_iter, server_file, copy_dest)) {
+        diff.matching_files.push_back(std::move(*client_iter));
+      } else {
+        diff.changed_files.emplace_back(server_file, std::move(*client_iter));
+      }
+      ++client_iter;
+    } else if (order > 0) {
+      diff.extraneous_files.push_back(std::move(*server_iter));
+      ++server_iter;
+    } else if (FilesMatch(*client_iter, *server_iter, copy_dest)) {
+      diff.matching_files.push_back(std::move(*client_iter));
+      ++client_iter;
+      ++server_iter;
+    } else {
+      diff.changed_files.emplace_back(*server_iter, std::move(*client_iter));
+      ++client_iter;
+      ++server_iter;
+    }
+  }
+
+  // Compare the arrays, sorting the dirs into the right buckets.
+  std::vector<DirInfo>::iterator client_dir_iter = client_dirs.begin();
+  std::vector<DirInfo>::iterator server_dir_iter = server_dirs.begin();
+
+  while (client_dir_iter != client_dirs.end() ||
+         server_dir_iter != server_dirs.end()) {
+    const int order =
+        client_dir_iter == client_dirs.end() ? 1  // Extraneous.
+        : server_dir_iter == server_dirs.end()
+            ? -1  // Missing.
+            : client_dir_iter->filepath.compare(server_dir_iter->filepath);
+
+    if (order < 0) {
+      diff.missing_dirs.push_back(std::move(*client_dir_iter));
+      ++client_dir_iter;
+    } else if (order > 0) {
+      diff.extraneous_dirs.push_back(std::move(*server_dir_iter));
+      ++server_dir_iter;
+    } else {
+      // Matching dirs in the copy_dest directory need to be created in the
+      // destination.
+      if (!copy_dest.empty() && server_dir_iter->base_dir == copy_dest.c_str())
+        diff.missing_dirs.push_back(std::move(*client_dir_iter));
+      else
+        diff.matching_dirs.push_back(std::move(*client_dir_iter));
+      ++client_dir_iter;
+      ++server_dir_iter;
+    }
+  }
+
+  // Remove all extraneous files and dirs from the |copy_dest| directory.
+  // Those should not be deleted with the --delete option.
+  if (!copy_dest.empty()) {
+    diff.extraneous_files.erase(
+        std::remove_if(diff.extraneous_files.begin(),
+                       diff.extraneous_files.end(),
+                       [&copy_dest](const FileInfo& dir) {
+                         return copy_dest.c_str() == dir.base_dir;
+                       }),
+        diff.extraneous_files.end());
+
+    diff.extraneous_dirs.erase(
+        std::remove_if(diff.extraneous_dirs.begin(), diff.extraneous_dirs.end(),
+                       [&copy_dest](const DirInfo& dir) {
+                         return copy_dest.c_str() == dir.base_dir;
+                       }),
+        diff.extraneous_dirs.end());
+  }
+
+  // Release memory.
+  std::vector<FileInfo> empty_client_files;
+  std::vector<FileInfo> empty_server_files;
+  client_files.swap(empty_client_files);
+  server_files.swap(empty_server_files);
+  std::vector<DirInfo> empty_client_dirs;
+  std::vector<DirInfo> empty_server_dirs;
+  client_dirs.swap(empty_client_dirs);
+  server_dirs.swap(empty_server_dirs);
+
+  return diff;
+}
+
+SendFileStatsResponse AdjustToFlagsAndGetStats(bool existing, bool checksum,
+                                               bool whole_file, Result* diff) {
+  // Record stats.
+  SendFileStatsResponse file_stats_response;
+  file_stats_response.set_num_missing_files(
+      static_cast<uint32_t>(diff->missing_files.size()));
+  file_stats_response.set_num_extraneous_files(
+      static_cast<uint32_t>(diff->extraneous_files.size()));
+  file_stats_response.set_num_matching_files(
+      static_cast<uint32_t>(diff->matching_files.size()));
+  file_stats_response.set_num_changed_files(
+      static_cast<uint32_t>(diff->changed_files.size()));
+  file_stats_response.set_num_missing_dirs(
+      static_cast<uint32_t>(diff->missing_dirs.size()));
+  file_stats_response.set_num_extraneous_dirs(
+      static_cast<uint32_t>(diff->extraneous_dirs.size()));
+  file_stats_response.set_num_matching_dirs(
+      static_cast<uint32_t>(diff->matching_dirs.size()));
+
+  // Take special flags into account that move files between the lists.
+
+  if (existing) {
+    // Do not copy missing files.
+    LOG_INFO("Removing missing files (--existing)");
+    std::vector<FileInfo> empty_files;
+    diff->missing_files.swap(empty_files);
+    std::vector<DirInfo> empty_dirs;
+    diff->missing_dirs.swap(empty_dirs);
+  }
+
+  if (checksum) {
+    // Move matching files over to changed files, so the delta-transfer
+    // algorithm is applied.
+    LOG_INFO("Moving matching files over to changed files (-c/--checksum)");
+    for (FileInfo& file : diff->matching_files)
+      diff->changed_files.emplace_back(file, std::move(file));
+    std::vector<FileInfo> empty;
+    diff->matching_files.swap(empty);
+  }
+
+  if (whole_file) {
+    // Move changed files over to the missing files, so they all get copied.
+    LOG_INFO("Moving changed files over to missing files (-W/--whole)");
+    for (ChangedFileInfo& file : diff->changed_files) {
+      diff->missing_files.emplace_back(
+          std::move(file.filepath), file.client_modified_time, file.client_size,
+          file.client_index, nullptr);
+    }
+    std::vector<ChangedFileInfo> empty;
+    diff->changed_files.swap(empty);
+  }
+
+  // Compute totals.
+  uint64_t total_missing_bytes = 0;
+  for (const FileInfo& file : diff->missing_files)
+    total_missing_bytes += file.size;
+
+  uint64_t total_changed_client_bytes = 0;
+  uint64_t total_changed_server_bytes = 0;
+  for (const ChangedFileInfo& file : diff->changed_files) {
+    total_changed_client_bytes += file.client_size;
+    total_changed_server_bytes += file.server_size;
+  }
+
+  // Set totals in stats. Note that the totals are computed from the MODIFIED
+  // file lists. This is important to get progress reporting right. The other
+  // stats, OTOH, are computed from the ORIGINAL file lists. They're displayed
+  // to the user.
+  file_stats_response.set_total_missing_bytes(total_missing_bytes);
+  file_stats_response.set_total_changed_client_bytes(
+      total_changed_client_bytes);
+  file_stats_response.set_total_changed_server_bytes(
+      total_changed_server_bytes);
+
+  return file_stats_response;
+}
+
+}  // namespace file_diff
+}  // namespace cdc_ft

cdc_rsync_server/file_diff_generator.h

@@ -0,0 +1,75 @@
+/*
+ * Copyright 2022 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef CDC_RSYNC_SERVER_FILE_DIFF_GENERATOR_H_
+#define CDC_RSYNC_SERVER_FILE_DIFF_GENERATOR_H_
+
+#include <vector>
+
+#include "cdc_rsync/protos/messages.pb.h"
+#include "cdc_rsync_server/file_info.h"
+
+namespace cdc_ft {
+namespace file_diff {
+
+struct Result {
+  // Files present on the client, but not on the server.
+  std::vector<FileInfo> missing_files;
+
+  // Files present on the server, but not on the client.
+  std::vector<FileInfo> extraneous_files;
+
+  // Files present on both, with different timestamp or file size.
+  std::vector<ChangedFileInfo> changed_files;
+
+  // Files present on both, with matching timestamp and file size.
+  std::vector<FileInfo> matching_files;
+
+  // Directories present on the client, but not on the server.
+  std::vector<DirInfo> missing_dirs;
+
+  // Directories present on the server, but not on the client.
+  std::vector<DirInfo> extraneous_dirs;
+
+  // Directories present on both client and server.
+  std::vector<DirInfo> matching_dirs;
+};
+
+// Generates the diff between
+// 1) |client_files| and |server_files| by comparing
+// file paths, modified times and file sizes.
+// If |double_check_missing| is true, missing files are checked for existence
+// (relative to |base_dir| and |copy_dest|, if non-empty)
+// before they are put into the missing bucket.
+// 2) |client_dirs| and |server_dirs| by comparing directory paths.
+// The passed-in vectors are consumed.
+Result Generate(std::vector<FileInfo>&& client_files,
+                std::vector<FileInfo>&& server_files,
+                std::vector<DirInfo>&& client_dirs,
+                std::vector<DirInfo>&& server_dirs, const std::string& base_dir,
+                const std::string& copy_dest, bool double_check_missing);
+
+// Adjusts file containers according to sync flags.
+// |existing|, |checksum|, |whole_file| are the sync flags, see command line
+// help. They cause files to be moved between containers.
+// |diff| is the result from Generate().
+SendFileStatsResponse AdjustToFlagsAndGetStats(bool existing, bool checksum,
+                                               bool whole_file, Result* diff);
+
+}  // namespace file_diff
+}  // namespace cdc_ft
+
+#endif  // CDC_RSYNC_SERVER_FILE_DIFF_GENERATOR_H_

cdc_rsync_server/file_diff_generator_test.cc

@@ -0,0 +1,619 @@
+// Copyright 2022 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "cdc_rsync_server/file_diff_generator.h"
+
+#include "cdc_rsync_server/file_info.h"
+#include "common/path.h"
+#include "common/status_test_macros.h"
+#include "common/test_main.h"
+#include "gtest/gtest.h"
+
+namespace cdc_ft {
+
+bool operator==(const FileInfo& a, const FileInfo& b) {
+  return a.filepath == b.filepath && a.modified_time == b.modified_time &&
+         a.size == b.size && a.client_index == b.client_index &&
+         a.base_dir == b.base_dir;
+}
+
+bool operator==(const ChangedFileInfo& a, const ChangedFileInfo& b) {
+  return a.filepath == b.filepath &&
+         a.client_modified_time == b.client_modified_time &&
+         a.client_size == b.client_size && a.server_size == b.server_size &&
+         a.client_index == b.client_index && a.base_dir == b.base_dir;
+}
+
+bool operator==(const DirInfo& a, const DirInfo& b) {
+  return a.filepath == b.filepath && a.client_index == b.client_index &&
+         a.base_dir == b.base_dir;
+}
+
+namespace {
+
+constexpr int64_t kModTime1 = 123;
+constexpr int64_t kModTime2 = 234;
+constexpr int64_t kModTime3 = 345;
+constexpr int64_t kModTime4 = 456;
+
+constexpr uint64_t kFileSize1 = 1000;
+constexpr uint64_t kFileSize2 = 2000;
+constexpr uint64_t kFileSize3 = 3000;
+constexpr uint64_t kFileSize4 = 4000;
+
+constexpr uint32_t kClientIndex1 = 1;
+constexpr uint32_t kClientIndex2 = 2;
+constexpr uint32_t kClientIndex3 = 3;
+constexpr uint32_t kClientIndex4 = 4;
+constexpr uint32_t kClientIndex5 = 5;
+
+constexpr bool kDoubleCheckMissing = true;
+constexpr bool kNoDoubleCheckMissing = false;
+
+constexpr bool kExisting = true;
+constexpr bool kNoExisting = false;
+
+constexpr bool kChecksum = true;
+constexpr bool kNoChecksum = false;
+
+constexpr bool kWholeFile = true;
+constexpr bool kNoWholeFile = false;
+
+constexpr char kNoCopyDest[] = "";
+
+// Note: FileDiffGenerator is a server-only class and only runs on GGP, but the
+// code is independent of the platform, so we can test it from Windows.
+class FileDiffGeneratorTest : public ::testing::Test {
+ protected:
+  std::string base_dir_ =
+      path::Join(GetTestDataDir("file_diff_generator"), "base_dir");
+
+  std::string copy_dest_ =
+      path::Join(GetTestDataDir("file_diff_generator"), "copy_dest");
+
+  const FileInfo client_file_ =
+      FileInfo("file/path1", kModTime1, kFileSize1, kClientIndex1, nullptr);
+  const FileInfo server_file_ =
+      FileInfo("file/path2", kModTime2, kFileSize2, FileInfo::kInvalidIndex,
+               base_dir_.c_str());
+
+  FileInfo matching_client_file_ =
+      FileInfo("file/path3", kModTime3, kFileSize3, kClientIndex2, nullptr);
+  const FileInfo matching_server_file_ =
+      FileInfo("file/path3", kModTime3, kFileSize3, FileInfo::kInvalidIndex,
+               base_dir_.c_str());
+
+  FileInfo changed_size_client_file_ =
+      FileInfo("file/path4", kModTime3, kFileSize3, kClientIndex3, nullptr);
+  const FileInfo changed_size_server_file_ =
+      FileInfo("file/path4", kModTime3, kFileSize4, FileInfo::kInvalidIndex,
+               base_dir_.c_str());
+
+  FileInfo changed_time_client_file_ =
+      FileInfo("file/path5", kModTime3, kFileSize3, kClientIndex3, nullptr);
+  const FileInfo changed_time_server_file_ =
+      FileInfo("file/path5", kModTime4, kFileSize3, FileInfo::kInvalidIndex,
+               base_dir_.c_str());
+
+  const DirInfo client_dir_ = DirInfo("dir/dir1", kClientIndex4, nullptr);
+  const DirInfo server_dir_ =
+      DirInfo("dir/dir2", FileInfo::kInvalidIndex, base_dir_.c_str());
+
+  const DirInfo matching_client_dir_ =
+      DirInfo("dir/dir3", kClientIndex5, nullptr);
+  const DirInfo matching_server_dir_ =
+      DirInfo("dir/dir3", FileInfo::kInvalidIndex, base_dir_.c_str());
+
+  // Creates a FileInfo struct by filling data from the file at
+  // |fi_base_dir|\|filename|. If |fi_base_dir| is nullptr (for client files)
+  // reads from |base_dir_| instead.
+  FileInfo CreateFileInfo(const char* filename, const char* fi_base_dir) {
+    std::string path =
+        path::Join(fi_base_dir ? fi_base_dir : base_dir_, filename);
+    path::Stats stats;
+    EXPECT_OK(path::GetStats(path, &stats));
+    return FileInfo(filename, stats.modified_time, stats.size, 0, fi_base_dir);
+  }
+
+  // Creates a default file_diff::Result with one file/dir in each bucket.
+  file_diff::Result MakeResultForAdjustTests() {
+    file_diff::Result diff;
+
+    diff.matching_files.push_back(matching_client_file_);
+    diff.changed_files.push_back(ChangedFileInfo(
+        changed_size_server_file_, FileInfo(changed_size_client_file_)));
+    diff.missing_files.push_back(client_file_);
+    diff.extraneous_files.push_back(server_file_);
+
+    diff.matching_dirs.push_back(matching_client_dir_);
+    diff.missing_dirs.push_back(client_dir_);
+    diff.extraneous_dirs.push_back(server_dir_);
+
+    return diff;
+  }
+};
+
+TEST_F(FileDiffGeneratorTest, MissingFile) {
+  file_diff::Result diff =
+      file_diff::Generate({client_file_}, {}, {}, {}, base_dir_, kNoCopyDest,
+                          kNoDoubleCheckMissing);
+
+  EXPECT_EQ(diff.missing_files, std::vector<FileInfo>({client_file_}));
+  EXPECT_TRUE(diff.extraneous_files.empty());
+  EXPECT_TRUE(diff.changed_files.empty());
+  EXPECT_TRUE(diff.matching_files.empty());
+  EXPECT_TRUE(diff.matching_dirs.empty());
+  EXPECT_TRUE(diff.missing_dirs.empty());
+  EXPECT_TRUE(diff.extraneous_dirs.empty());
+}
+
+TEST_F(FileDiffGeneratorTest, ExtraneousFile) {
+  file_diff::Result diff =
+      file_diff::Generate({}, {server_file_}, {}, {}, base_dir_, kNoCopyDest,
+                          kNoDoubleCheckMissing);
+
+  EXPECT_TRUE(diff.missing_files.empty());
+  EXPECT_EQ(diff.extraneous_files, std::vector<FileInfo>({server_file_}));
+  EXPECT_TRUE(diff.changed_files.empty());
+  EXPECT_TRUE(diff.matching_files.empty());
+  EXPECT_TRUE(diff.matching_dirs.empty());
+  EXPECT_TRUE(diff.missing_dirs.empty());
+  EXPECT_TRUE(diff.extraneous_dirs.empty());
+}
+
+TEST_F(FileDiffGeneratorTest, MatchingFiles) {
+  file_diff::Result diff =
+      file_diff::Generate({matching_client_file_}, {matching_server_file_}, {},
+                          {}, base_dir_, kNoCopyDest, kNoDoubleCheckMissing);
+
+  EXPECT_TRUE(diff.missing_files.empty());
+  EXPECT_TRUE(diff.extraneous_files.empty());
+  EXPECT_TRUE(diff.changed_files.empty());
+  EXPECT_EQ(diff.matching_files,
+            std::vector<FileInfo>({matching_client_file_}));
+  EXPECT_TRUE(diff.matching_dirs.empty());
+  EXPECT_TRUE(diff.missing_dirs.empty());
+  EXPECT_TRUE(diff.extraneous_dirs.empty());
+}
+
+TEST_F(FileDiffGeneratorTest, ChangedFiles) {
+  // Purposely swap the order for the server files to test sorting.
+  file_diff::Result diff = file_diff::Generate(
+      {changed_time_client_file_, changed_size_client_file_},
+      {changed_size_server_file_, changed_time_server_file_}, {}, {}, base_dir_,
+      kNoCopyDest, kNoDoubleCheckMissing);
+
+  EXPECT_TRUE(diff.missing_files.empty());
+  EXPECT_TRUE(diff.extraneous_files.empty());
+  EXPECT_EQ(diff.changed_files,
+            std::vector<ChangedFileInfo>(
+                {ChangedFileInfo(changed_size_server_file_,
+                                 std::move(changed_size_client_file_)),
+                 ChangedFileInfo(changed_time_server_file_,
+                                 std::move(changed_time_client_file_))}));
+  EXPECT_TRUE(diff.matching_files.empty());
+  EXPECT_TRUE(diff.matching_dirs.empty());
+  EXPECT_TRUE(diff.missing_dirs.empty());
+  EXPECT_TRUE(diff.extraneous_dirs.empty());
+}
+
+TEST_F(FileDiffGeneratorTest, OrderIndependence) {
+  std::vector<FileInfo> client_files = {client_file_, matching_client_file_,
+                                        changed_size_client_file_,
+                                        changed_time_client_file_};
+  std::vector<FileInfo> server_files = {server_file_, matching_server_file_,
+                                        changed_size_server_file_,
+                                        changed_time_server_file_};
+
+  std::vector<FileInfo> expected_missing_files = {client_file_};
+  std::vector<FileInfo> expected_extraneous_files = {server_file_};
+  std::vector<ChangedFileInfo> expected_changed_files = {
+      ChangedFileInfo(changed_size_server_file_,
+                      std::move(changed_size_client_file_)),
+      ChangedFileInfo(changed_time_server_file_,
+                      std::move(changed_time_client_file_))};
+  std::vector<FileInfo> expected_matching_files = {matching_client_file_};
+
+  // Make several tests, each time with |server_files| permuted a bit..
+  for (size_t backwards = 0; backwards < 2; ++backwards) {
+    for (size_t circular = 0; circular < server_files.size(); ++circular) {
+      file_diff::Result diff =
+          file_diff::Generate(std::vector<FileInfo>(client_files),
+                              std::vector<FileInfo>(server_files), {}, {},
+                              base_dir_, kNoCopyDest, kNoDoubleCheckMissing);
+
+      EXPECT_EQ(diff.missing_files, expected_missing_files);
+      EXPECT_EQ(diff.extraneous_files, expected_extraneous_files);
+      EXPECT_EQ(diff.changed_files, expected_changed_files);
+      EXPECT_EQ(diff.matching_files, expected_matching_files);
+
+      // Circular permutation.
+      server_files.insert(server_files.begin(), server_files.back());
+      server_files.pop_back();
+    }
+
+    // Reverse order.
+    std::reverse(server_files.begin(), server_files.end());
+  }
+}
+
+TEST_F(FileDiffGeneratorTest, DoubleCheckMissing_NoCopyDest) {
+  // file_a is matching the real file, file_b is changed, file_h is missing.
+  FileInfo file_a = CreateFileInfo("a.txt", nullptr);
+  FileInfo file_b = CreateFileInfo("b.txt", nullptr);
+  file_b.modified_time = 0;
+  FileInfo file_h("h.txt", 0, 0, 0, nullptr);
+
+  file_diff::Result diff =
+      file_diff::Generate({file_a, file_b, file_h}, {}, {}, {}, base_dir_,
+                          kNoCopyDest, kDoubleCheckMissing);
+
+  FileInfo server_file_a = CreateFileInfo("a.txt", base_dir_.c_str());
+  FileInfo server_file_b = CreateFileInfo("b.txt", base_dir_.c_str());
+
+  ChangedFileInfo changed_file_b(server_file_b, std::move(file_b));
+
+  EXPECT_EQ(diff.matching_files, std::vector<FileInfo>({file_a}));
+  EXPECT_EQ(diff.changed_files, std::vector<ChangedFileInfo>({changed_file_b}));
+  EXPECT_EQ(diff.missing_files, std::vector<FileInfo>({file_h}));
+  EXPECT_TRUE(diff.extraneous_files.empty());
+}
+
+TEST_F(FileDiffGeneratorTest, DoubleCheckMissing_CopyDest) {
+  // Tests all permutations of client files and server files in base_dir as
+  // well as copy_dest. Special treatment is marked as !!!.
+  //    client files        server files                 resulting diff list
+  //                 base_dir          copy_dest
+  // a  exists       exists, matching  missing           matching/base_dir
+  // b  exists       exists, changed   missing           changed/base_dir
+  // c  missing      exists            missing           extraneous/base_dir
+  // d  exists       missing           missing           missing
+  // e  exists       exists, matching  exists, ignored   matching/base_dir
+  // f  exists       exists, changed   exists, ignored   changed/base_dir
+  // g  missing      exists            exists, ignored   extraneous/base_dir
+  // h  exists       missing           exists, matching  changed/copy_dest (!!!)
+  // i  exists       missing           exists, changed   changed/copy_dest
+  // j  missing      missing           exists            ignored (!!!)
+
+  // Client files.
+  FileInfo file_a = CreateFileInfo("a.txt", nullptr);
+  FileInfo file_b = CreateFileInfo("b.txt", nullptr);
+  // c missing
+  FileInfo file_d = FileInfo("d.txt", 0, 0, 0, nullptr);
+  FileInfo file_e = CreateFileInfo("e.txt", nullptr);
+  FileInfo file_f = CreateFileInfo("f.txt", nullptr);
+  // g missing
+  FileInfo file_h = CreateFileInfo("h.txt", copy_dest_.c_str());
+  file_h.base_dir = nullptr;
+  FileInfo file_i = FileInfo("i.txt", 0, 0, 0, nullptr);
+  // j missing
+
+  // Mark files b and f as changed.
+  file_b.modified_time = 0;
+  file_f.modified_time = 0;
+
+  std::vector<FileInfo> client_files = {file_a, file_b, file_d, file_e,
+                                        file_f, file_h, file_i};
+
+  // Server files in base_dir. d, h, i and j are missing.
+  FileInfo server_file_a = CreateFileInfo("a.txt", base_dir_.c_str());
+  FileInfo server_file_b = CreateFileInfo("b.txt", base_dir_.c_str());
+  FileInfo server_file_c = CreateFileInfo("c.txt", base_dir_.c_str());
+  FileInfo server_file_e = CreateFileInfo("e.txt", base_dir_.c_str());
+  FileInfo server_file_f = CreateFileInfo("f.txt", base_dir_.c_str());
+  FileInfo server_file_g = CreateFileInfo("g.txt", base_dir_.c_str());
+
+  std::vector<FileInfo> server_files = {server_file_a, server_file_b,
+                                        server_file_c, server_file_e,
+                                        server_file_f, server_file_g};
+
+  std::vector<FileInfo> expected_matching_files = {file_a, file_e};
+
+  ChangedFileInfo changed_file_b(server_file_b, std::move(file_b));
+  ChangedFileInfo changed_file_f(server_file_f, std::move(file_f));
+  ChangedFileInfo changed_file_h(CreateFileInfo("h.txt", copy_dest_.c_str()),
+                                 std::move(file_h));
+  ChangedFileInfo changed_file_i(CreateFileInfo("i.txt", copy_dest_.c_str()),
+                                 std::move(file_i));
+  std::vector<ChangedFileInfo> expected_changed_files = {
+      changed_file_b, changed_file_f, changed_file_h, changed_file_i};
+
+  std::vector<FileInfo> expected_missing_files = {file_d};
+  std::vector<FileInfo> expected_extraneous_files = {server_file_c,
+                                                     server_file_g};
+
+  // The server files in copy_dest are stat'ed by file_diff::Generate().
+
+  file_diff::Result diff =
+      file_diff::Generate(std::move(client_files), std::move(server_files), {},
+                          {}, base_dir_, copy_dest_, kDoubleCheckMissing);
+
+  EXPECT_EQ(diff.matching_files, expected_matching_files);
+  EXPECT_EQ(diff.changed_files, expected_changed_files);
+  EXPECT_EQ(diff.missing_files, expected_missing_files);
+  EXPECT_EQ(diff.extraneous_files, expected_extraneous_files);
+}
+
+TEST_F(FileDiffGeneratorTest, MissingDir) {
+  file_diff::Result diff = file_diff::Generate(
+      {}, {}, {client_dir_}, {}, base_dir_, kNoCopyDest, kNoDoubleCheckMissing);
+
+  EXPECT_EQ(diff.missing_dirs, std::vector<DirInfo>({client_dir_}));
+  EXPECT_TRUE(diff.extraneous_dirs.empty());
+  EXPECT_TRUE(diff.matching_dirs.empty());
+  EXPECT_TRUE(diff.matching_files.empty());
+  EXPECT_TRUE(diff.missing_files.empty());
+  EXPECT_TRUE(diff.changed_files.empty());
+  EXPECT_TRUE(diff.extraneous_files.empty());
+}
+
+TEST_F(FileDiffGeneratorTest, ExtraneousDir) {
+  file_diff::Result diff = file_diff::Generate(
+      {}, {}, {}, {server_dir_}, base_dir_, kNoCopyDest, kNoDoubleCheckMissing);
+
+  EXPECT_TRUE(diff.missing_dirs.empty());
+  EXPECT_EQ(diff.extraneous_dirs, std::vector<DirInfo>({server_dir_}));
+  EXPECT_TRUE(diff.matching_dirs.empty());
+  EXPECT_TRUE(diff.matching_files.empty());
+  EXPECT_TRUE(diff.missing_files.empty());
+  EXPECT_TRUE(diff.changed_files.empty());
+  EXPECT_TRUE(diff.extraneous_files.empty());
+}
+
+TEST_F(FileDiffGeneratorTest, MatchingDirs) {
+  file_diff::Result diff = file_diff::Generate(
+      {}, {}, {matching_client_dir_}, {matching_server_dir_}, base_dir_,
+      kNoCopyDest, kNoDoubleCheckMissing);
+
+  EXPECT_TRUE(diff.missing_dirs.empty());
+  EXPECT_TRUE(diff.extraneous_dirs.empty());
+  EXPECT_EQ(diff.matching_dirs, std::vector<DirInfo>({matching_client_dir_}));
+  EXPECT_TRUE(diff.matching_files.empty());
+  EXPECT_TRUE(diff.missing_files.empty());
+  EXPECT_TRUE(diff.changed_files.empty());
+  EXPECT_TRUE(diff.extraneous_files.empty());
+}
+
+TEST_F(FileDiffGeneratorTest, DirOrderIndependence) {
+  std::vector<DirInfo> client_dirs = {client_dir_, matching_client_dir_};
+  std::vector<DirInfo> server_dirs = {server_dir_, matching_server_dir_};
+
+  std::vector<DirInfo> expected_missing_dirs = {client_dir_};
+  std::vector<DirInfo> expected_extraneous_dirs = {server_dir_};
+  std::vector<DirInfo> expected_matching_dirs = {matching_client_dir_};
+
+  // Make several tests, each time with |server_dirs| permuted a bit.
+  for (size_t backwards = 0; backwards < 2; ++backwards) {
+    for (size_t circular = 0; circular < server_dirs.size(); ++circular) {
+      file_diff::Result diff =
+          file_diff::Generate({}, {}, std::vector<DirInfo>(client_dirs),
+                              std::vector<DirInfo>(server_dirs), base_dir_,
+                              kNoCopyDest, kNoDoubleCheckMissing);
+
+      EXPECT_EQ(diff.missing_dirs, expected_missing_dirs);
+      EXPECT_EQ(diff.extraneous_dirs, expected_extraneous_dirs);
+      EXPECT_EQ(diff.matching_dirs, expected_matching_dirs);
+
+      // Circular permutation.
+      server_dirs.insert(server_dirs.begin(), server_dirs.back());
+      server_dirs.pop_back();
+    }
+
+    // Reverse order.
+    std::reverse(server_dirs.begin(), server_dirs.end());
+  }
+}
+
+TEST_F(FileDiffGeneratorTest, CopyDest_Dirs) {
+  DirInfo client_dir1("dir/dir1", kClientIndex1, nullptr);
+  DirInfo client_dir2("dir/dir2", kClientIndex2, nullptr);
+
+  // Matching in |copy_dest_|
+  // -> counted as missing (so it gets created in destination)
+  DirInfo server_dir1("dir/dir1", FileInfo::kInvalidIndex, copy_dest_.c_str());
+  // Extraneous in |copy_dest_|
+  // -> ignored (shouldn't delete dirs in package dir!)
+  DirInfo server_dir2("dir/dir3", FileInfo::kInvalidIndex, copy_dest_.c_str());
+
+  file_diff::Result diff = file_diff::Generate(
+      {}, {}, std::vector<DirInfo>{client_dir1, client_dir2},
+      std::vector<DirInfo>{server_dir1, server_dir2}, base_dir_, copy_dest_,
+      kNoDoubleCheckMissing);
+
+  EXPECT_EQ(diff.missing_dirs,
+            std::vector<DirInfo>({client_dir1, client_dir2}));
+  EXPECT_TRUE(diff.extraneous_dirs.empty());
+  EXPECT_TRUE(diff.matching_dirs.empty());
+}
+
+TEST_F(FileDiffGeneratorTest, Adjust_DefaultParams) {
+  file_diff::Result diff = MakeResultForAdjustTests();
+  SendFileStatsResponse response = file_diff::AdjustToFlagsAndGetStats(
+      kNoExisting, kNoChecksum, kNoWholeFile, &diff);
+
+  EXPECT_EQ(diff.matching_files,
+            std::vector<FileInfo>({matching_client_file_}));
+  EXPECT_EQ(diff.missing_files, std::vector<FileInfo>({client_file_}));
+  EXPECT_EQ(
+      diff.changed_files,
+      std::vector<ChangedFileInfo>({ChangedFileInfo(
+          changed_size_server_file_, std::move(changed_size_client_file_))}));
+  EXPECT_EQ(diff.extraneous_files, std::vector<FileInfo>({server_file_}));
+
+  EXPECT_EQ(diff.matching_dirs, std::vector<DirInfo>({matching_client_dir_}));
+  EXPECT_EQ(diff.missing_dirs, std::vector<DirInfo>({client_dir_}));
+  EXPECT_EQ(diff.extraneous_dirs, std::vector<DirInfo>({server_dir_}));
+
+  EXPECT_EQ(response.num_matching_files(), 1);
+  EXPECT_EQ(response.num_missing_files(), 1);
+  EXPECT_EQ(response.num_changed_files(), 1);
+  EXPECT_EQ(response.num_extraneous_files(), 1);
+
+  EXPECT_EQ(response.num_matching_dirs(), 1);
+  EXPECT_EQ(response.num_missing_dirs(), 1);
+  EXPECT_EQ(response.num_extraneous_dirs(), 1);
+
+  EXPECT_EQ(response.total_changed_client_bytes(),
+            changed_size_client_file_.size);
+  EXPECT_EQ(response.total_changed_server_bytes(),
+            changed_size_server_file_.size);
+  EXPECT_EQ(response.total_missing_bytes(), client_file_.size);
+}
+
+TEST_F(FileDiffGeneratorTest, Adjust_Existing) {
+  file_diff::Result diff = MakeResultForAdjustTests();
+  SendFileStatsResponse response = file_diff::AdjustToFlagsAndGetStats(
+      kExisting, kNoChecksum, kNoWholeFile, &diff);
+
+  // Existing removes missing files.
+  EXPECT_EQ(diff.matching_files,
+            std::vector<FileInfo>({matching_client_file_}));
+  EXPECT_TRUE(diff.missing_files.empty());
+  EXPECT_EQ(
+      diff.changed_files,
+      std::vector<ChangedFileInfo>({ChangedFileInfo(
+          changed_size_server_file_, std::move(changed_size_client_file_))}));
+  EXPECT_EQ(diff.extraneous_files, std::vector<FileInfo>({server_file_}));
+
+  EXPECT_EQ(diff.matching_dirs, std::vector<DirInfo>({matching_client_dir_}));
+  EXPECT_TRUE(diff.missing_dirs.empty());
+  EXPECT_EQ(diff.extraneous_dirs, std::vector<DirInfo>({server_dir_}));
+
+  // These stats should be unchanged.
+  EXPECT_EQ(response.num_matching_files(), 1);
+  EXPECT_EQ(response.num_missing_files(), 1);
+  EXPECT_EQ(response.num_changed_files(), 1);
+  EXPECT_EQ(response.num_extraneous_files(), 1);
+
+  EXPECT_EQ(response.num_matching_dirs(), 1);
+  EXPECT_EQ(response.num_missing_dirs(), 1);
+  EXPECT_EQ(response.num_extraneous_dirs(), 1);
+
+  // These stats should be computed from the actual containers.
+  EXPECT_EQ(response.total_changed_client_bytes(),
+            changed_size_client_file_.size);
+  EXPECT_EQ(response.total_changed_server_bytes(),
+            changed_size_server_file_.size);
+  EXPECT_EQ(response.total_missing_bytes(), 0);
+}
+
+TEST_F(FileDiffGeneratorTest, Adjust_Checksum) {
+  file_diff::Result diff = MakeResultForAdjustTests();
+  SendFileStatsResponse response = file_diff::AdjustToFlagsAndGetStats(
+      kNoExisting, kChecksum, kNoWholeFile, &diff);
+
+  // Checksum moves matching files to changed files.
+  EXPECT_TRUE(diff.matching_files.empty());
+  EXPECT_EQ(diff.missing_files, std::vector<FileInfo>({client_file_}));
+  EXPECT_EQ(diff.changed_files,
+            std::vector<ChangedFileInfo>(
+                {ChangedFileInfo(changed_size_server_file_,
+                                 std::move(changed_size_client_file_)),
+                 ChangedFileInfo(matching_client_file_,
+                                 std::move(matching_client_file_))}));
+  EXPECT_EQ(diff.extraneous_files, std::vector<FileInfo>({server_file_}));
+
+  EXPECT_EQ(diff.matching_dirs, std::vector<DirInfo>({matching_client_dir_}));
+  EXPECT_EQ(diff.missing_dirs, std::vector<DirInfo>({client_dir_}));
+  EXPECT_EQ(diff.extraneous_dirs, std::vector<DirInfo>({server_dir_}));
+
+  // These stats should be unchanged.
+  EXPECT_EQ(response.num_matching_files(), 1);
+  EXPECT_EQ(response.num_missing_files(), 1);
+  EXPECT_EQ(response.num_changed_files(), 1);
+  EXPECT_EQ(response.num_extraneous_files(), 1);
+
+  EXPECT_EQ(response.num_matching_dirs(), 1);
+  EXPECT_EQ(response.num_missing_dirs(), 1);
+  EXPECT_EQ(response.num_extraneous_dirs(), 1);
+
+  // These stats should be computed from the actual containers.
+  EXPECT_EQ(response.total_changed_client_bytes(),
+            changed_size_client_file_.size + matching_client_file_.size);
+  EXPECT_EQ(response.total_changed_server_bytes(),
+            changed_size_server_file_.size + matching_client_file_.size);
+  EXPECT_EQ(response.total_missing_bytes(), client_file_.size);
+}
+
+TEST_F(FileDiffGeneratorTest, Adjust_WholeFile) {
+  file_diff::Result diff = MakeResultForAdjustTests();
+  SendFileStatsResponse response = file_diff::AdjustToFlagsAndGetStats(
+      kNoExisting, kNoChecksum, kWholeFile, &diff);
+
+  // WholeFile moves changed files to missing files.
+  EXPECT_EQ(diff.matching_files,
+            std::vector<FileInfo>({matching_client_file_}));
+  EXPECT_EQ(diff.missing_files,
+            std::vector<FileInfo>({client_file_, changed_size_client_file_}));
+  EXPECT_TRUE(diff.changed_files.empty());
+  EXPECT_EQ(diff.extraneous_files, std::vector<FileInfo>({server_file_}));
+
+  EXPECT_EQ(diff.matching_dirs, std::vector<DirInfo>({matching_client_dir_}));
+  EXPECT_EQ(diff.missing_dirs, std::vector<DirInfo>({client_dir_}));
+  EXPECT_EQ(diff.extraneous_dirs, std::vector<DirInfo>({server_dir_}));
+
+  // These stats should be unchanged.
+  EXPECT_EQ(response.num_matching_files(), 1);
+  EXPECT_EQ(response.num_missing_files(), 1);
+  EXPECT_EQ(response.num_changed_files(), 1);
+  EXPECT_EQ(response.num_extraneous_files(), 1);
+
+  EXPECT_EQ(response.num_matching_dirs(), 1);
+  EXPECT_EQ(response.num_missing_dirs(), 1);
+  EXPECT_EQ(response.num_extraneous_dirs(), 1);
+
+  // These stats should be computed from the actual containers.
+  EXPECT_EQ(response.total_changed_client_bytes(), 0);
+  EXPECT_EQ(response.total_changed_server_bytes(), 0);
+  EXPECT_EQ(response.total_missing_bytes(),
+            client_file_.size + changed_size_client_file_.size);
+}
+
+TEST_F(FileDiffGeneratorTest, Adjust_ChecksumAndWholeFile) {
+  file_diff::Result diff = MakeResultForAdjustTests();
+  SendFileStatsResponse response = file_diff::AdjustToFlagsAndGetStats(
+      kNoExisting, kChecksum, kWholeFile, &diff);
+
+  // Checksum+WholeFile moves both matching and changed files to missing files.
+  EXPECT_TRUE(diff.matching_files.empty());
+  EXPECT_EQ(diff.missing_files,
+            std::vector<FileInfo>({client_file_, changed_size_client_file_,
+                                   matching_client_file_}));
+  EXPECT_TRUE(diff.changed_files.empty());
+  EXPECT_EQ(diff.extraneous_files, std::vector<FileInfo>({server_file_}));
+
+  EXPECT_EQ(diff.matching_dirs, std::vector<DirInfo>({matching_client_dir_}));
+  EXPECT_EQ(diff.missing_dirs, std::vector<DirInfo>({client_dir_}));
+  EXPECT_EQ(diff.extraneous_dirs, std::vector<DirInfo>({server_dir_}));
+
+  // These stats should be unchanged.
+  EXPECT_EQ(response.num_matching_files(), 1);
+  EXPECT_EQ(response.num_missing_files(), 1);
+  EXPECT_EQ(response.num_changed_files(), 1);
+  EXPECT_EQ(response.num_extraneous_files(), 1);
+
+  EXPECT_EQ(response.num_matching_dirs(), 1);
+  EXPECT_EQ(response.num_missing_dirs(), 1);
+  EXPECT_EQ(response.num_extraneous_dirs(), 1);
+
+  // These stats should be computed from the actual containers.
+  EXPECT_EQ(response.total_changed_client_bytes(), 0);
+  EXPECT_EQ(response.total_changed_server_bytes(), 0);
+  EXPECT_EQ(response.total_missing_bytes(), client_file_.size +
+                                                changed_size_client_file_.size +
+                                                matching_client_file_.size);
+}
+
+}  // namespace
+}  // namespace cdc_ft

cdc_rsync_server/file_finder.cc

@@ -0,0 +1,96 @@
+// Copyright 2022 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "cdc_rsync_server/file_finder.h"
+
+#include "common/path.h"
+#include "common/path_filter.h"
+#include "common/platform.h"
+#include "common/status.h"
+
+namespace cdc_ft {
+
+FileFinder::FileFinder() {}
+
+absl::Status FileFinder::AddFiles(const std::string& base_dir, bool recursive,
+                                  PathFilter* path_filter) {
+  std::vector<FileInfo>* files = &files_;
+  std::vector<DirInfo>* dirs = &dirs_;
+  auto handler = [files, dirs, &base_dir, path_filter](
+                     std::string dir, std::string filename,
+                     int64_t modified_time, uint64_t size, bool is_directory) {
+    std::string path = path::Join(dir.substr(base_dir.size()), filename);
+    if (path_filter->IsMatch(path)) {
+      if (is_directory) {
+        dirs->emplace_back(path, FileInfo::kInvalidIndex, base_dir.c_str());
+      } else {
+        files->emplace_back(path, modified_time, size, FileInfo::kInvalidIndex,
+                            base_dir.c_str());
+      }
+    }
+    return absl::OkStatus();
+  };
+
+#if PLATFORM_WINDOWS
+  // SearchFiles needs a wildcard on Windows. Currently only used for tests.
+  absl::Status status =
+      path::SearchFiles(path::Join(base_dir, "*"), recursive, handler);
+#elif PLATFORM_LINUX
+  absl::Status status = path::SearchFiles(base_dir, recursive, handler);
+#endif
+  if (!status.ok()) {
+    return WrapStatus(status, "Failed to find files for '%s'", base_dir);
+  }
+
+  return absl::OkStatus();
+}
+
+void FileFinder::ReleaseFiles(std::vector<FileInfo>* files,
+                              std::vector<DirInfo>* dirs) {
+  assert(files);
+  assert(dirs);
+
+  // Dedupe files and directories. Note that the combination of std::stable_sort
+  // and std::unique is guaranteed to keep the entries added first to the lists.
+  // In practice, this kicks out the element in the copy_dest directory (e.g.
+  // the package) and keeps the one in the destination (e.g. /mnt/developer), if
+  // both are present.
+  std::stable_sort(files_.begin(), files_.end(),
+                   [](const FileInfo& a, const FileInfo& b) {
+                     return a.filepath < b.filepath;
+                   });
+  std::stable_sort(dirs_.begin(), dirs_.end(),
+                   [](const DirInfo& a, const DirInfo& b) {
+                     return a.filepath < b.filepath;
+                   });
+
+  files_.erase(std::unique(files_.begin(), files_.end(),
+                           [](const FileInfo& a, const FileInfo& b) {
+                             return a.filepath == b.filepath;
+                           }),
+               files_.end());
+
+  dirs_.erase(std::unique(dirs_.begin(), dirs_.end(),
+                          [](const DirInfo& a, const DirInfo& b) {
+                            return a.filepath == b.filepath;
+                          }),
+              dirs_.end());
+
+  *files = std::move(files_);
+  *dirs = std::move(dirs_);
+}
+
+FileFinder::~FileFinder() = default;
+
+}  // namespace cdc_ft

cdc_rsync_server/file_finder.h

@@ -0,0 +1,53 @@
+/*
+ * Copyright 2022 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef CDC_RSYNC_SERVER_FILE_FINDER_H_
+#define CDC_RSYNC_SERVER_FILE_FINDER_H_
+
+#include <vector>
+
+#include "absl/status/status.h"
+#include "cdc_rsync_server/file_info.h"
+
+namespace cdc_ft {
+
+class PathFilter;
+
+// Scans directories and gathers contained files and directories.
+class FileFinder {
+ public:
+  FileFinder();
+  ~FileFinder();
+
+  // Gathers files and directories in |base_dir|. If |recursive| is true,
+  // searches recursively. |path_filter| is used to filter files and
+  // directories.
+  // If subsequent calls to AddFiles find a file or directory with the same
+  // relative path, this file or directory is ignored.
+  absl::Status AddFiles(const std::string& base_dir, bool recursive,
+                        PathFilter* path_filter);
+
+  // Returns all found files and directories.
+  void ReleaseFiles(std::vector<FileInfo>* files, std::vector<DirInfo>* dirs);
+
+ private:
+  std::vector<FileInfo> files_;
+  std::vector<DirInfo> dirs_;
+};
+
+}  // namespace cdc_ft
+
+#endif  // CDC_RSYNC_SERVER_FILE_FINDER_H_

cdc_rsync_server/file_finder_test.cc

@@ -0,0 +1,121 @@
+// Copyright 2022 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "cdc_rsync_server/file_finder.h"
+
+#include "common/log.h"
+#include "common/path.h"
+#include "common/path_filter.h"
+#include "common/status_test_macros.h"
+#include "common/test_main.h"
+#include "gtest/gtest.h"
+
+namespace cdc_ft {
+namespace {
+
+constexpr bool kNonRecursive = false;
+constexpr bool kRecursive = true;
+
+class FileFinderTest : public ::testing::Test {
+ public:
+  void SetUp() override {
+    Log::Initialize(std::make_unique<ConsoleLog>(LogLevel::kInfo));
+  }
+
+  void TearDown() override { Log::Shutdown(); }
+
+ protected:
+  std::string base_dir_ =
+      path::Join(GetTestDataDir("file_finder"), path::ToNative("base_dir/"));
+
+  std::string copy_dest_ =
+      path::Join(GetTestDataDir("file_finder"), path::ToNative("copy_dest/"));
+
+  template <typename PathClass>
+  static void ExpectMatch(
+      const std::vector<PathClass>& paths,
+      std::vector<std::pair<std::string, std::string>> base_dir_and_rel_path) {
+    EXPECT_EQ(base_dir_and_rel_path.size(), paths.size());
+    if (base_dir_and_rel_path.size() != paths.size()) return;
+
+    for (size_t n = 0; n < paths.size(); ++n) {
+      EXPECT_EQ(paths[n].base_dir, base_dir_and_rel_path[n].first);
+      EXPECT_EQ(paths[n].filepath, base_dir_and_rel_path[n].second);
+    }
+  }
+
+  PathFilter path_filter_;
+  std::vector<FileInfo> files_;
+  std::vector<DirInfo> dirs_;
+};
+
+TEST_F(FileFinderTest, FindSucceedsInvalidPath) {
+  // Invalid paths are just ignored.
+  std::string invalid_path = path::Join(base_dir_, "invalid");
+  FileFinder finder;
+  EXPECT_OK(finder.AddFiles(invalid_path, kNonRecursive, &path_filter_));
+  finder.ReleaseFiles(&files_, &dirs_);
+  EXPECT_TRUE(files_.empty());
+  EXPECT_TRUE(dirs_.empty());
+}
+
+TEST_F(FileFinderTest, FindSucceedsNonRecursive) {
+  FileFinder finder;
+  EXPECT_OK(finder.AddFiles(base_dir_, kNonRecursive, &path_filter_));
+  finder.ReleaseFiles(&files_, &dirs_);
+  ExpectMatch(files_, {{base_dir_, "a.txt"}, {base_dir_, "b.txt"}});
+  ExpectMatch(dirs_, {{base_dir_, "dir1"}, {base_dir_, "dir2"}});
+}
+
+TEST_F(FileFinderTest, FindSucceedsRecursive) {
+  FileFinder finder;
+  EXPECT_OK(finder.AddFiles(base_dir_, kRecursive, &path_filter_));
+  finder.ReleaseFiles(&files_, &dirs_);
+  ExpectMatch(files_, {{base_dir_, "a.txt"},
+                       {base_dir_, "b.txt"},
+                       {base_dir_, path::ToNative("dir1/c.txt")},
+                       {base_dir_, path::ToNative("dir2/d.txt")}});
+  ExpectMatch(dirs_, {{base_dir_, "dir1"}, {base_dir_, "dir2"}});
+}
+
+TEST_F(FileFinderTest, FindSucceedsRecursiveWithCopyDest) {
+  FileFinder finder;
+  EXPECT_OK(finder.AddFiles(base_dir_, kRecursive, &path_filter_));
+  EXPECT_OK(finder.AddFiles(copy_dest_, kRecursive, &path_filter_));
+  finder.ReleaseFiles(&files_, &dirs_);
+  ExpectMatch(files_, {{base_dir_, "a.txt"},
+                       {base_dir_, "b.txt"},
+                       {base_dir_, path::ToNative("dir1/c.txt")},
+                       {copy_dest_, path::ToNative("dir1/f.txt")},
+                       {base_dir_, path::ToNative("dir2/d.txt")},
+                       {copy_dest_, path::ToNative("dir3/d.txt")},
+                       {copy_dest_, "e.txt"}});
+  ExpectMatch(dirs_,
+              {{base_dir_, "dir1"}, {base_dir_, "dir2"}, {copy_dest_, "dir3"}});
+}
+
+TEST_F(FileFinderTest, FindSucceedsWithFilter) {
+  path_filter_.AddRule(PathFilter::Rule::Type::kExclude, "a.txt");
+
+  FileFinder finder;
+  EXPECT_OK(finder.AddFiles(base_dir_, kRecursive, &path_filter_));
+  finder.ReleaseFiles(&files_, &dirs_);
+  ExpectMatch(files_, {{base_dir_, "b.txt"},
+                       {base_dir_, path::ToNative("dir1/c.txt")},
+                       {base_dir_, path::ToNative("dir2/d.txt")}});
+  ExpectMatch(dirs_, {{base_dir_, "dir1"}, {base_dir_, "dir2"}});
+}
+
+}  // namespace
+}  // namespace cdc_ft

cdc_rsync_server/file_info.h

@@ -0,0 +1,83 @@
+/*
+ * Copyright 2022 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef CDC_RSYNC_SERVER_FILE_INFO_H_
+#define CDC_RSYNC_SERVER_FILE_INFO_H_
+
+#include <string>
+
+namespace cdc_ft {
+
+struct FileInfo {
+  static constexpr uint32_t kInvalidIndex = UINT32_MAX;
+
+  // Path relative to |base_dir|.
+  std::string filepath;
+  int64_t modified_time;
+  uint64_t size;
+  // For client files: Index into the client file list.
+  uint32_t client_index;
+  // For server files: Base directory. If nullptr, |filepath| is assumed to be
+  // relative to the destination directory.
+  const char* base_dir;
+
+  FileInfo(std::string filepath, int64_t modified_time, uint64_t size,
+           uint32_t client_index, const char* base_dir)
+      : filepath(std::move(filepath)),
+        modified_time(modified_time),
+        size(size),
+        client_index(client_index),
+        base_dir(base_dir) {}
+};
+
+struct DirInfo {
+  static constexpr uint32_t kInvalidIndex = UINT32_MAX;
+
+  // Path relative to |base_dir|.
+  std::string filepath;
+  uint32_t client_index;
+  // For server files: Base directory. If nullptr, |filepath| is assumed to be
+  // relative to the destination directory.
+  const char* base_dir;
+
+  DirInfo(std::string filepath, uint32_t client_index, const char* base_dir)
+      : filepath(std::move(filepath)),
+        client_index(client_index),
+        base_dir(base_dir) {}
+};
+
+// Similar to FileInfo, but size is needed from both client and server.
+struct ChangedFileInfo {
+  std::string filepath;
+  int64_t client_modified_time;
+  uint64_t client_size;
+  uint64_t server_size;
+  uint32_t client_index;
+  const char* base_dir;
+
+  // Moves |client_file| data into this class.
+  ChangedFileInfo(const FileInfo& server_file, FileInfo&& client_file)
+      : filepath(std::move(client_file.filepath)),
+        client_modified_time(client_file.modified_time),
+        client_size(client_file.size),
+        server_size(server_file.size),
+        client_index(client_file.client_index),
+        base_dir(server_file.base_dir) {}
+};
+
+}  // namespace cdc_ft
+
+#endif  // CDC_RSYNC_SERVER_FILE_INFO_H_

cdc_rsync_server/main.cc

@@ -0,0 +1,105 @@
+// Copyright 2022 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "cdc_rsync/base/server_exit_code.h"
+#include "cdc_rsync_server/cdc_rsync_server.h"
+#include "common/gamelet_component.h"
+#include "common/log.h"
+#include "common/status.h"
+
+namespace {
+
+void SendErrorMessage(const char* msg) {
+  constexpr char marker = cdc_ft::kServerErrorMarker;
+  fprintf(stderr, "%c%s%c", marker, msg, marker);
+}
+
+}  // namespace
+
+namespace cdc_ft {
+
+// Returns custom error codes based on the tag associated with |status|. This is
+// used to display custom error messages on the client.
+// Example: A bind failure usually means two instances are in use
+//          simultaneously.
+ServerExitCode GetExitCode(const absl::Status& status) {
+  absl::optional<Tag> tag = GetTag(status);
+  if (!tag.has_value()) {
+    return kServerExitCodeGeneric;
+  }
+
+  // Some tags translate to a special error message on the client.
+  switch (tag.value()) {
+    case Tag::kAddressInUse:
+      // Can't bind port, probably two instances in use simultaneously.
+      return kServerExitCodeAddressInUse;
+
+    case Tag::kSocketEof:
+      // Usually means client disconnected and shut down already.
+    case Tag::kDeployServer:
+    case Tag::kInstancePickerNotAvailableInQuietMode:
+    case Tag::kConnectionTimeout:
+    case Tag::kCount:
+      // Should not happen in server.
+      break;
+  }
+
+  return kServerExitCodeGeneric;
+}
+
+}  // namespace cdc_ft
+
+int main(int argc, const char** argv) {
+  if (argc < 2) {
+    printf("Usage: cdc_rsync_server <port> cdc_rsync_server <size> <time> \n");
+    printf("       where <size> and <time> are the file size and modified\n");
+    printf("       timestamp (Unix epoch) of the corresponding component.\n");
+    return cdc_ft::kServerExitCodeGenericStartup;
+  }
+
+  int port = atoi(argv[1]);
+  if (port == 0) {
+    SendErrorMessage(absl::StrFormat("Invalid port '%s'", argv[1]).c_str());
+    return cdc_ft::kServerExitCodeGenericStartup;
+  }
+
+  // The rest is expected to be sets of gamelet component info consisting of
+  // (filename, filesize, modified_time). This is used check whether the
+  // components are up-to-date.
+  std::vector<cdc_ft::GameletComponent> components =
+      cdc_ft::GameletComponent::FromCommandLineArgs(argc - 2, argv + 2);
+
+  cdc_ft::Log::Initialize(
+      std::make_unique<cdc_ft::ConsoleLog>(cdc_ft::LogLevel::kWarning));
+  cdc_ft::GgpRsyncServer server;
+  if (!server.CheckComponents(components)) {
+    return cdc_ft::kServerExitCodeOutOfDate;
+  }
+
+  absl::Status status = server.Run(port);
+  if (status.ok()) {
+    return 0;
+  }
+
+  cdc_ft::ServerExitCode code = cdc_ft::GetExitCode(status);
+
+  // Print full error in verbose mode, so that it's not lost.
+  if (server.GetVerbosity() >= 2) {
+    fprintf(stderr, "Server error: %s\n", status.ToString().c_str());
+  }
+
+  // Send error message to the client and return code.
+  SendErrorMessage(std::string(status.message()).c_str());
+  return code;
+}

cdc_rsync_server/server_socket.cc

@@ -0,0 +1,214 @@
+// Copyright 2022 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "cdc_rsync_server/server_socket.h"
+
+#include <netinet/in.h>
+#include <sys/socket.h>
+#include <unistd.h>
+
+#include <cerrno>
+
+#include "common/log.h"
+#include "common/status.h"
+
+namespace cdc_ft {
+
+namespace {
+
+int kInvalidFd = -1;
+
+// Keep re-evaluating the expression |x| while it returns EINTR.
+#define HANDLE_EINTR(x)                                     \
+  ({                                                        \
+    decltype(x) eintr_wrapper_result;                       \
+    do {                                                    \
+      eintr_wrapper_result = (x);                           \
+    } while (eintr_wrapper_result == -1 && errno == EINTR); \
+    eintr_wrapper_result;                                   \
+  })
+
+}  // namespace
+
+ServerSocket::ServerSocket()
+    : Socket(), listen_sockfd_(kInvalidFd), conn_sockfd_(kInvalidFd) {}
+
+ServerSocket::~ServerSocket() {
+  Disconnect();
+  StopListening();
+}
+
+absl::Status ServerSocket::StartListening(int port) {
+  if (listen_sockfd_ != kInvalidFd) {
+    return MakeStatus("Already listening");
+  }
+
+  LOG_DEBUG("Open socket");
+  listen_sockfd_ = socket(AF_INET, SOCK_STREAM, 0);
+  if (listen_sockfd_ < 0) {
+    listen_sockfd_ = kInvalidFd;
+    return MakeStatus("socket() failed: %s", strerror(errno));
+  }
+
+  // If the program terminates abnormally, the socket might remain in a
+  // TIME_WAIT state and report "address already in use" on bind(). Setting
+  // SO_REUSEADDR works around that. See
+  // https://hea-www.harvard.edu/~fine/Tech/addrinuse.html
+  int enable = 1;
+  if (setsockopt(listen_sockfd_, SOL_SOCKET, SO_REUSEADDR, &enable,
+                 sizeof(enable)) < 0) {
+    LOG_DEBUG("setsockopt() failed");
+  }
+
+  LOG_DEBUG("Bind socket");
+  sockaddr_in serv_addr;
+  memset(&serv_addr, 0, sizeof(serv_addr));
+  serv_addr.sin_family = AF_INET;
+  serv_addr.sin_addr.s_addr = INADDR_ANY;
+  serv_addr.sin_port = htons(port);
+  if (bind(listen_sockfd_, (struct sockaddr*)&serv_addr, sizeof(serv_addr)) <
+      0) {
+    absl::Status status =
+        MakeStatus("bind() to port %i failed: %s", port, strerror(errno));
+    if (errno == EADDRINUSE) {
+      // Happens when two instances are run at the same time. Help callers to
+      // print reasonable errors.
+      status = SetTag(status, Tag::kAddressInUse);
+    }
+    close(listen_sockfd_);
+    listen_sockfd_ = kInvalidFd;
+
+    return status;
+  }
+
+  LOG_DEBUG("Listen");
+  listen(listen_sockfd_, 1);
+  return absl::OkStatus();
+}
+
+void ServerSocket::StopListening() {
+  if (listen_sockfd_ != kInvalidFd) {
+    close(listen_sockfd_);
+    listen_sockfd_ = kInvalidFd;
+  }
+
+  LOG_INFO("Stopped listening.");
+}
+
+absl::Status ServerSocket::WaitForConnection() {
+  if (conn_sockfd_ != kInvalidFd) {
+    return MakeStatus("Already connected");
+  }
+
+  sockaddr_in cli_addr;
+  socklen_t cli_len = sizeof(cli_addr);
+  conn_sockfd_ = accept(listen_sockfd_, (struct sockaddr*)&cli_addr, &cli_len);
+  if (conn_sockfd_ < 0) {
+    conn_sockfd_ = kInvalidFd;
+    return MakeStatus("accept() failed: %s", strerror(errno));
+  }
+
+  LOG_DEBUG("Client connected");
+  return absl::OkStatus();
+}
+
+void ServerSocket::Disconnect() {
+  if (conn_sockfd_ != kInvalidFd) {
+    close(conn_sockfd_);
+    conn_sockfd_ = kInvalidFd;
+  }
+
+  LOG_INFO("Disconnected");
+}
+
+absl::Status ServerSocket::ShutdownSendingEnd() {
+  int result = shutdown(conn_sockfd_, SHUT_WR);
+  if (result != 0) {
+    return MakeStatus("shutdown() failed: %s", strerror(errno));
+  }
+
+  return absl::OkStatus();
+}
+
+absl::Status ServerSocket::Send(const void* buffer, size_t size) {
+  const uint8_t* curr_ptr = reinterpret_cast<const uint8_t*>(buffer);
+  ssize_t bytes_left = size;
+  while (bytes_left > 0) {
+    ssize_t bytes_written =
+        HANDLE_EINTR(send(conn_sockfd_, curr_ptr, bytes_left, /*flags*/ 0));
+
+    if (bytes_written < 0) {
+      if (errno == EAGAIN || errno == EWOULDBLOCK) {
+        // Shouldn't happen as the socket should be blocking.
+        LOG_DEBUG("Socket would block");
+        continue;
+      }
+
+      return MakeStatus("write() to fd %i failed: %s", conn_sockfd_,
+                        strerror(errno));
+    }
+
+    bytes_left -= bytes_written;
+    curr_ptr += bytes_written;
+  }
+  return absl::OkStatus();
+}
+
+absl::Status ServerSocket::Receive(void* buffer, size_t size,
+                                   bool allow_partial_read,
+                                   size_t* bytes_received) {
+  *bytes_received = 0;
+  if (size == 0) {
+    return absl::OkStatus();
+  }
+
+  uint8_t* curr_ptr = reinterpret_cast<uint8_t*>(buffer);
+  ssize_t bytes_left = size;
+  while (bytes_left > 0) {
+    ssize_t bytes_read =
+        HANDLE_EINTR(recv(conn_sockfd_, curr_ptr, bytes_left, /*flags*/ 0));
+
+    if (bytes_read < 0) {
+      if (errno == EAGAIN || errno == EWOULDBLOCK) {
+        // Shouldn't happen as the socket should be blocking.
+        LOG_DEBUG("Socket would block");
+        continue;
+      }
+
+      return MakeStatus("recv() from fd %i failed: %s", conn_sockfd_,
+                        strerror(errno));
+    }
+
+    bytes_left -= bytes_read;
+    *bytes_received += bytes_read;
+    curr_ptr += bytes_read;
+
+    if (bytes_read == 0) {
+      // EOF. Make sure we're not in the middle of a message.
+      if (bytes_left < static_cast<ssize_t>(size)) {
+        return MakeStatus("EOF after partial read");
+      }
+
+      LOG_DEBUG("EOF() detected");
+      return SetTag(MakeStatus("EOF detected"), Tag::kSocketEof);
+    }
+
+    if (allow_partial_read) {
+      break;
+    }
+  }
+  return absl::OkStatus();
+}
+
+}  // namespace cdc_ft

cdc_rsync_server/server_socket.h

@@ -0,0 +1,62 @@
+/*
+ * Copyright 2022 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef CDC_RSYNC_SERVER_SERVER_SOCKET_H_
+#define CDC_RSYNC_SERVER_SERVER_SOCKET_H_
+
+#include "absl/status/status.h"
+#include "cdc_rsync/base/socket.h"
+
+namespace cdc_ft {
+
+class ServerSocket : public Socket {
+ public:
+  ServerSocket();
+  ~ServerSocket();
+
+  // Starts listening for connections on |port|.
+  absl::Status StartListening(int port);
+
+  // Stops listening for connections. No-op if already stopped/never started.
+  void StopListening();
+
+  // Waits for a client to connect. Only supports one connection. Repeating
+  // the call with an existing connection results in an error.
+  absl::Status WaitForConnection();
+
+  // Disconnects the client. No-op if not connected.
+  void Disconnect();
+
+  // Shuts down the sending end of the socket. This will interrupt any receive
+  // calls on the client and shut it down.
+  absl::Status ShutdownSendingEnd();
+
+  // Socket:
+  absl::Status Send(const void* buffer, size_t size) override;
+  absl::Status Receive(void* buffer, size_t size, bool allow_partial_read,
+                       size_t* bytes_received) override;
+
+ private:
+  // Listening socket file descriptor (where new connections are accepted).
+  int listen_sockfd_;
+
+  // Connection socket file descriptor (where data is sent to/received from).
+  int conn_sockfd_;
+};
+
+}  // namespace cdc_ft
+
+#endif  // CDC_RSYNC_SERVER_SERVER_SOCKET_H_

cdc_rsync_server/testdata/file_diff_generator/base_dir/a.txt

@@ -0,0 +1 @@
+aaa

cdc_rsync_server/testdata/file_diff_generator/base_dir/b.txt

@@ -0,0 +1 @@
+bbb

cdc_rsync_server/testdata/file_diff_generator/base_dir/c.txt

@@ -0,0 +1 @@
+ccc

cdc_rsync_server/testdata/file_diff_generator/base_dir/e.txt

@@ -0,0 +1 @@
+eee

cdc_rsync_server/testdata/file_diff_generator/base_dir/f.txt

@@ -0,0 +1 @@
+fff

cdc_rsync_server/testdata/file_diff_generator/base_dir/g.txt

@@ -0,0 +1 @@
+ggg

cdc_rsync_server/testdata/file_diff_generator/copy_dest/e.txt

@@ -0,0 +1 @@
+eee

cdc_rsync_server/testdata/file_diff_generator/copy_dest/f.txt

@@ -0,0 +1 @@
+fff

cdc_rsync_server/testdata/file_diff_generator/copy_dest/g.txt

@@ -0,0 +1 @@
+ggg

cdc_rsync_server/testdata/file_diff_generator/copy_dest/h.txt

@@ -0,0 +1 @@
+hhh

cdc_rsync_server/testdata/file_diff_generator/copy_dest/i.txt

@@ -0,0 +1 @@
+iii

cdc_rsync_server/testdata/file_diff_generator/copy_dest/j.txt

@@ -0,0 +1 @@
+jjj

cdc_rsync_server/testdata/file_finder/base_dir/a.txt

@@ -0,0 +1 @@
+aaa

cdc_rsync_server/testdata/file_finder/base_dir/b.txt

@@ -0,0 +1 @@
+bbb

cdc_rsync_server/testdata/file_finder/base_dir/dir1/c.txt

@@ -0,0 +1 @@
+ccc

cdc_rsync_server/testdata/file_finder/base_dir/dir2/d.txt

@@ -0,0 +1 @@
+ddd

cdc_rsync_server/testdata/file_finder/copy_dest/a.txt

@@ -0,0 +1 @@
+aaa

cdc_rsync_server/testdata/file_finder/copy_dest/dir1/c.txt

@@ -0,0 +1 @@
+ccc

cdc_rsync_server/testdata/file_finder/copy_dest/dir1/f.txt

@@ -0,0 +1 @@
+fff

cdc_rsync_server/testdata/file_finder/copy_dest/dir3/d.txt

@@ -0,0 +1 @@
+ddd

cdc_rsync_server/testdata/file_finder/copy_dest/e.txt

@@ -0,0 +1 @@
+eee

cdc_rsync_server/unzstd_stream.cc

@@ -0,0 +1,89 @@
+// Copyright 2022 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "cdc_rsync_server/unzstd_stream.h"
+
+#include "cdc_rsync/base/socket.h"
+#include "common/status.h"
+
+namespace cdc_ft {
+
+UnzstdStream::UnzstdStream(Socket* socket) : socket_(socket) {
+  init_status_ =
+      WrapStatus(Initialize(), "Failed to initialize stream decompressor");
+}
+
+UnzstdStream::~UnzstdStream() {
+  if (dctx_) {
+    ZSTD_freeDCtx(dctx_);
+    dctx_ = nullptr;
+  }
+}
+
+absl::Status UnzstdStream::Read(void* out_buffer, size_t out_size,
+                                size_t* bytes_read, bool* eof) {
+  *bytes_read = 0;
+  *eof = false;
+  if (!init_status_.ok()) {
+    return init_status_;
+  }
+
+  ZSTD_outBuffer output = {out_buffer, out_size, 0};
+  while (output.pos < output.size && !*eof) {
+    if (input_.pos == input_.size) {
+      // Read more compressed input data.
+      // Allow partial reads since the stream could end any time.
+      size_t in_size;
+      absl::Status status =
+          socket_->Receive(in_buffer_.data(), in_buffer_.size(),
+                           /*allow_partial_read=*/true, &in_size);
+      if (!status.ok()) {
+        return WrapStatus(status, "socket_->ReceiveEx() failed");
+      }
+      input_.pos = 0;
+      input_.size = in_size;
+    }
+
+    // Decompress.
+    size_t ret = ZSTD_decompressStream(dctx_, &output, &input_);
+    if (ZSTD_isError(ret)) {
+      return MakeStatus("Failed to decompress data: %s",
+                        ZSTD_getErrorName(ret));
+    }
+
+    *eof = (ret == 0);
+    if (*eof && input_.pos < input_.size) {
+      return MakeStatus("EOF with %u bytes input data available",
+                        input_.size - input_.pos);
+    }
+  }
+
+  // Output buffer is full or eof.
+  *bytes_read = output.pos;
+  return absl::OkStatus();
+}
+
+absl::Status UnzstdStream::Initialize() {
+  dctx_ = ZSTD_createDCtx();
+  if (!dctx_) {
+    return MakeStatus("Decompression context creation failed");
+  }
+
+  in_buffer_.resize(ZSTD_DStreamInSize());
+  input_ = {in_buffer_.data(), 0, 0};
+
+  return absl::OkStatus();
+}
+
+}  // namespace cdc_ft

cdc_rsync_server/unzstd_stream.h

@@ -0,0 +1,50 @@
+/*
+ * Copyright 2022 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef CDC_RSYNC_SERVER_UNZSTD_STREAM_H_
+#define CDC_RSYNC_SERVER_UNZSTD_STREAM_H_
+
+#include "absl/status/status.h"
+#include "cdc_rsync/base/message_pump.h"
+#include "lib/zstd.h"
+
+namespace cdc_ft {
+
+class Socket;
+
+// Streaming decompression using zstd.
+class UnzstdStream : public MessagePump::InputReader {
+ public:
+  explicit UnzstdStream(Socket* socket);
+  virtual ~UnzstdStream();
+
+  // MessagePump::InputReader:
+  absl::Status Read(void* out_buffer, size_t out_size, size_t* bytes_read,
+                    bool* eof) override;
+
+ private:
+  absl::Status Initialize();
+
+  Socket* const socket_;
+  std::vector<uint8_t> in_buffer_;
+  ZSTD_inBuffer input_;
+  ZSTD_DCtx* dctx_;
+  absl::Status init_status_;
+};
+
+}  // namespace cdc_ft
+
+#endif  // CDC_RSYNC_SERVER_UNZSTD_STREAM_H_