Releasing the former Stadia file transfer tools

The tools allow efficient and fast synchronization of large directory
trees from a Windows workstation to a Linux target machine.

cdc_rsync* support efficient copy of files by using content-defined
chunking (CDC) to identify chunks within files that can be reused.

asset_stream_manager + cdc_fuse_fs support efficient streaming of a
local directory to a remote virtual file system based on FUSE. It also
employs CDC to identify and reuse unchanged data chunks.

cdc_rsync/.gitignore

@@ -0,0 +1,4 @@
+x64/*
+generated_protos
+*.log
+*.user

cdc_rsync/BUILD

@@ -0,0 +1,191 @@
+load(
+    "//tools:windows_cc_library.bzl",
+    "cc_windows_shared_library",
+)
+
+package(default_visibility = [
+    "//:__subpackages__",
+])
+
+cc_library(
+    name = "client_file_info",
+    hdrs = ["client_file_info.h"],
+)
+
+cc_library(
+    name = "client_socket",
+    srcs = ["client_socket.cc"],
+    hdrs = ["client_socket.h"],
+    target_compatible_with = ["@platforms//os:windows"],
+    deps = [
+        "//cdc_rsync/base:socket",
+        "//common:log",
+        "//common:status",
+        "//common:util",
+    ],
+)
+
+cc_library(
+    name = "file_finder_and_sender",
+    srcs = ["file_finder_and_sender.cc"],
+    hdrs = ["file_finder_and_sender.h"],
+    target_compatible_with = ["@platforms//os:windows"],
+    deps = [
+        ":client_file_info",
+        "//cdc_rsync/base:message_pump",
+        "//cdc_rsync/protos:messages_cc_proto",
+        "//common:log",
+        "//common:path",
+        "//common:path_filter",
+        "//common:platform",
+        "//common:util",
+    ],
+)
+
+cc_test(
+    name = "file_finder_and_sender_test",
+    srcs = ["file_finder_and_sender_test.cc"],
+    data = ["testdata/root.txt"] + glob(["testdata/file_finder_and_sender/**"]),
+    deps = [
+        ":file_finder_and_sender",
+        "//cdc_rsync/base:fake_socket",
+        "//cdc_rsync/protos:messages_cc_proto",
+        "//common:status_test_macros",
+        "//common:test_main",
+        "@com_google_googletest//:gtest",
+        "@com_google_protobuf//:protobuf_lite",
+    ],
+)
+
+cc_windows_shared_library(
+    name = "cdc_rsync",
+    srcs = [
+        "cdc_rsync.cc",
+        "cdc_rsync_client.cc",
+        "dllmain.cc",
+    ],
+    hdrs = [
+        "cdc_rsync.h",
+        "cdc_rsync_client.h",
+        "error_messages.h",
+    ],
+    linkopts = select({
+        "//tools:windows": [
+            "/DEFAULTLIB:Ws2_32.lib",  # Sockets, e.g. recv, send, WSA*.
+        ],
+        "//conditions:default": [],
+    }),
+    local_defines = ["COMPILING_DLL"],
+    target_compatible_with = ["@platforms//os:windows"],
+    deps = [
+        ":client_socket",
+        ":file_finder_and_sender",
+        ":parallel_file_opener",
+        ":progress_tracker",
+        ":zstd_stream",
+        "//cdc_rsync/base:cdc_interface",
+        "//cdc_rsync/base:message_pump",
+        "//cdc_rsync/base:server_exit_code",
+        "//cdc_rsync/base:socket",
+        "//cdc_rsync/protos:messages_cc_proto",
+        "//common:gamelet_component",
+        "//common:log",
+        "//common:path",
+        "//common:path_filter",
+        "//common:platform",
+        "//common:port_manager",
+        "//common:process",
+        "//common:remote_util",
+        "//common:sdk_util",
+        "//common:status",
+        "//common:status_macros",
+        "//common:threadpool",
+        "//common:util",
+        "@com_google_absl//absl/status",
+    ],
+)
+
+cc_library(
+    name = "parallel_file_opener",
+    srcs = ["parallel_file_opener.cc"],
+    hdrs = ["parallel_file_opener.h"],
+    data = ["testdata/root.txt"] + glob(["testdata/parallel_file_opener/**"]),
+    deps = [
+        ":client_file_info",
+        "//common:path",
+        "//common:platform",
+        "//common:threadpool",
+    ],
+)
+
+cc_test(
+    name = "parallel_file_opener_test",
+    srcs = ["parallel_file_opener_test.cc"],
+    deps = [
+        ":parallel_file_opener",
+        "//common:test_main",
+        "@com_google_googletest//:gtest",
+    ],
+)
+
+cc_library(
+    name = "progress_tracker",
+    srcs = ["progress_tracker.cc"],
+    hdrs = ["progress_tracker.h"],
+    deps = [
+        ":file_finder_and_sender",
+        "//cdc_rsync/base:cdc_interface",
+        "//common:stopwatch",
+        "@com_github_jsoncpp//:jsoncpp",
+        "@com_google_absl//absl/strings:str_format",
+    ],
+)
+
+cc_test(
+    name = "progress_tracker_test",
+    srcs = ["progress_tracker_test.cc"],
+    deps = [
+        ":progress_tracker",
+        "//cdc_rsync/protos:messages_cc_proto",
+        "//common:test_main",
+        "//common:testing_clock",
+        "@com_google_googletest//:gtest",
+    ],
+)
+
+cc_library(
+    name = "zstd_stream",
+    srcs = ["zstd_stream.cc"],
+    hdrs = ["zstd_stream.h"],
+    deps = [
+        ":client_socket",
+        "//common:buffer",
+        "//common:status",
+        "//common:status_macros",
+        "//common:stopwatch",
+        "@com_github_zstd//:zstd",
+    ],
+)
+
+cc_test(
+    name = "zstd_stream_test",
+    srcs = ["zstd_stream_test.cc"],
+    deps = [
+        ":zstd_stream",
+        "//cdc_rsync/base:fake_socket",
+        "//cdc_rsync_server:unzstd_stream",
+        "//common:status_test_macros",
+        "//common:test_main",
+        "@com_github_zstd//:zstd",
+    ],
+)
+
+filegroup(
+    name = "all_test_sources",
+    srcs = glob(["*_test.cc"]),
+)
+
+filegroup(
+    name = "all_test_data",
+    srcs = glob(["testdata/**"]),
+)

cdc_rsync/README.md

@@ -0,0 +1,5 @@
+# CDC RSync
+
+CDC RSync is a command line tool / library for uploading files to a remote machine in an rsync-like
+fashion. It quickly skips files with matching timestamp and size, and only transfers deltas for
+existing files.

cdc_rsync/base/BUILD

@@ -0,0 +1,92 @@
+package(default_visibility = [
+    "//:__subpackages__",
+])
+
+cc_library(
+    name = "cdc_interface",
+    srcs = ["cdc_interface.cc"],
+    hdrs = ["cdc_interface.h"],
+    deps = [
+        ":message_pump",
+        "//cdc_rsync/protos:messages_cc_proto",
+        "//common:buffer",
+        "//common:log",
+        "//common:path",
+        "//common:status",
+        "//common:threadpool",
+        "//fastcdc",
+        "@com_github_blake3//:blake3",
+        "@com_google_absl//absl/strings:str_format",
+    ],
+)
+
+cc_test(
+    name = "cdc_interface_test",
+    srcs = ["cdc_interface_test.cc"],
+    data = ["testdata/root.txt"] + glob(["testdata/cdc_interface/**"]),
+    deps = [
+        ":cdc_interface",
+        ":fake_socket",
+        "//common:status_test_macros",
+        "//common:test_main",
+        "@com_google_googletest//:gtest",
+    ],
+)
+
+cc_library(
+    name = "fake_socket",
+    srcs = ["fake_socket.cc"],
+    hdrs = ["fake_socket.h"],
+    deps = [
+        "//cdc_rsync/base:socket",
+        "@com_google_absl//absl/status",
+    ],
+)
+
+cc_library(
+    name = "message_pump",
+    srcs = ["message_pump.cc"],
+    hdrs = ["message_pump.h"],
+    deps = [
+        ":socket",
+        "//common:buffer",
+        "//common:log",
+        "//common:status",
+        "@com_google_absl//absl/status",
+        "@com_google_absl//absl/strings:str_format",
+        "@com_google_protobuf//:protobuf_lite",
+    ],
+)
+
+cc_test(
+    name = "message_pump_test",
+    srcs = ["message_pump_test.cc"],
+    deps = [
+        ":fake_socket",
+        ":message_pump",
+        "//cdc_rsync/protos:messages_cc_proto",
+        "//common:status_test_macros",
+        "//common:test_main",
+        "@com_google_googletest//:gtest",
+    ],
+)
+
+cc_library(
+    name = "server_exit_code",
+    hdrs = ["server_exit_code.h"],
+)
+
+cc_library(
+    name = "socket",
+    hdrs = ["socket.h"],
+)
+
+filegroup(
+    name = "all_test_sources",
+    srcs = glob(["*_test.cc"]),
+)
+
+filegroup(
+    name = "all_test_data",
+    srcs = glob(["testdata/**"]),
+)

cdc_rsync/base/cdc_interface.cc

@@ -0,0 +1,670 @@
+// Copyright 2022 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "cdc_rsync/base/cdc_interface.h"
+
+#include <vector>
+
+#include "absl/strings/str_format.h"
+#include "blake3.h"
+#include "cdc_rsync/base/message_pump.h"
+#include "cdc_rsync/protos/messages.pb.h"
+#include "common/buffer.h"
+#include "common/path.h"
+#include "common/status.h"
+#include "common/util.h"
+#include "fastcdc/fastcdc.h"
+
+#if PLATFORM_LINUX
+#include <fcntl.h>
+#endif
+
+namespace cdc_ft {
+namespace {
+
+// The average chunk size should be as low as possible, but not too low.
+// Lower sizes mean better delta-encoding and hence less data uploads.
+// However, chunking becomes slower for lower sizes. At 8 KB, a gamelet can
+// still process close to 700 MB/sec, which matches hard drive speed.
+// Signature data rate is another factor. The gamelet generates signature data
+// at a rate of 700 MB/sec / kAvgChunkSize * sizeof(Chunk) = 1.7 MB/sec for 8 KB
+// chunks. That means, the client needs at least 16 MBit download bandwidth to
+// stream signatures or else this part becomes slower. 4 KB chunks would require
+// a 32 MBit connection.
+constexpr size_t kAvgChunkSize = 8 * 1024;
+constexpr size_t kMinChunkSize = kAvgChunkSize / 2;
+constexpr size_t kMaxChunkSize = kAvgChunkSize * 4;
+
+// This number was found by experimentally optimizing chunking throughput.
+constexpr size_t kFileIoBufferSize = kMaxChunkSize * 4;
+
+// Limits the size of contiguous patch chunks where data is copied from the
+// basis file. Necessary since the server copies chunks in one go and doesn't
+// split them up (would be possible, but unnecessarily complicates code).
+constexpr size_t kCombinedChunkSizeThreshold = 64 * 1024;
+
+// Number of hashing tasks in flight at a given point of time.
+constexpr size_t kMaxNumHashTasks = 64;
+
+#pragma pack(push, 1)
+// 16 byte hashes guarantee a sufficiently low chance of hash collisions. For
+// 8 byte the chance of a hash collision is actually quite high for large files
+// 0.0004% for a 100 GB file and 8 KB chunks.
+struct Hash {
+  uint64_t low;
+  uint64_t high;
+
+  bool operator==(const Hash& other) const {
+    return low == other.low && high == other.high;
+  }
+  bool operator!=(const Hash& other) const { return !(*this == other); }
+};
+#pragma pack(pop)
+
+static_assert(sizeof(Hash) <= BLAKE3_OUT_LEN, "");
+
+}  // namespace
+}  // namespace cdc_ft
+
+namespace std {
+
+template <>
+struct hash<cdc_ft::Hash> {
+  size_t operator()(const cdc_ft::Hash& hash) const { return hash.low; }
+};
+
+}  // namespace std
+
+namespace cdc_ft {
+namespace {
+
+// Send a batch of signatures every 8 MB of processed data (~90 packets per
+// second at 700 MB/sec processing rate). The size of each signature batch is
+// kMinNumChunksPerBatch * sizeof(Chunk), e.g. 20 KB for an avg chunk size of
+// 8 KB.
+constexpr int kMinSigBatchDataSize = 8 * 1024 * 1024;
+constexpr int kMinNumChunksPerBatch = kMinSigBatchDataSize / kAvgChunkSize;
+
+// Send patch commands in batches of at least that size for efficiency.
+constexpr int kPatchRequestSizeThreshold = 65536;
+
+// 16 bytes hash, 4 bytes size = 20 bytes.
+struct Chunk {
+  Hash hash;
+  uint32_t size = 0;
+  Chunk(const Hash& hash, uint32_t size) : hash(hash), size(size) {}
+};
+
+Hash ComputeHash(const void* data, size_t size) {
+  assert(data);
+  Hash hash;
+  blake3_hasher hasher;
+  blake3_hasher_init(&hasher);
+  blake3_hasher_update(&hasher, data, size);
+  blake3_hasher_finalize(&hasher, reinterpret_cast<uint8_t*>(&hash),
+                         sizeof(hash));
+  return hash;
+}
+
+// Task that computes hashes for a single chunk and adds the result to
+// AddSignaturesResponse.
+class HashTask : public Task {
+ public:
+  HashTask() {}
+  ~HashTask() {}
+
+  HashTask(const HashTask& other) = delete;
+  HashTask& operator=(HashTask&) = delete;
+
+  // Sets the data to compute the hash of.
+  // Should be called before queuing the task.
+  void SetData(const void* data, size_t size) {
+    buffer_.reserve(size);
+    buffer_.resize(size);
+    memcpy(buffer_.data(), data, size);
+  }
+
+  // Appends the computed hash to |response|.
+  // Should be called once the task is finished.
+  void AppendHash(AddSignaturesResponse* response) const {
+    response->add_sizes(static_cast<uint32_t>(buffer_.size()));
+    std::string* hashes = response->mutable_hashes();
+    hashes->append(reinterpret_cast<const char*>(&hash_), sizeof(hash_));
+  }
+
+  void ThreadRun(IsCancelledPredicate is_cancelled) override {
+    hash_ = ComputeHash(buffer_.data(), buffer_.size());
+  }
+
+ private:
+  Buffer buffer_;
+  struct Hash hash_ = {0};
+};
+
+class ServerChunkReceiver {
+ public:
+  explicit ServerChunkReceiver(MessagePump* message_pump)
+      : message_pump_(message_pump) {
+    assert(message_pump_);
+  }
+
+  // Receives server signature packets and places the data into a map
+  // (chunk hash) -> (server-side file offset).
+  // If |block| is false, returns immediately if no data is available.
+  // If |block| is true, blocks until some data is available.
+  // |num_server_bytes_processed| is set to the total size of the chunks
+  // received.
+  absl::Status Receive(bool block, uint64_t* num_server_bytes_processed) {
+    assert(num_server_bytes_processed);
+    *num_server_bytes_processed = 0;
+
+    // Already all server chunks received?
+    if (all_chunks_received_) {
+      return absl::OkStatus();
+    }
+
+    // If no data is available, early out (unless blocking is requested).
+    if (!block && !message_pump_->CanReceive()) {
+      return absl::OkStatus();
+    }
+
+    // Receive signatures.
+    AddSignaturesResponse response;
+    absl::Status status =
+        message_pump_->ReceiveMessage(PacketType::kAddSignatures, &response);
+    if (!status.ok()) {
+      return WrapStatus(status, "Failed to receive AddSignaturesResponse");
+    }
+
+    // Validate size of packed hashes, just in case.
+    const int num_chunks = response.sizes_size();
+    if (response.hashes().size() != num_chunks * sizeof(Hash)) {
+      return MakeStatus("Bad hashes size. Expected %u. Actual %u.",
+                        num_chunks * sizeof(Hash), response.hashes().size());
+    }
+
+    // An empty packet marks the end of the server chunks.
+    if (num_chunks == 0) {
+      all_chunks_received_ = true;
+      return absl::OkStatus();
+    }
+
+    // Copy the data over to |server_chunk_offsets|.
+    const Hash* hashes =
+        reinterpret_cast<const Hash*>(response.hashes().data());
+    for (int n = 0; n < num_chunks; ++n) {
+      uint32_t size = response.sizes(n);
+      chunk_offsets_.insert({hashes[n], curr_offset_});
+      curr_offset_ += size;
+      *num_server_bytes_processed += size;
+    }
+
+    return absl::OkStatus();
+  }
+
+  // True if all server chunks have been received.
+  bool AllChunksReceived() const { return all_chunks_received_; }
+
+  // Returns a map (server chunk hash) -> (offset of that chunk in server file).
+  const std::unordered_map<Hash, uint64_t>& ChunkOffsets() const {
+    return chunk_offsets_;
+  }
+
+ private:
+  MessagePump* message_pump_;
+
+  // Maps server chunk hashes to the file offset in the server file.
+  std::unordered_map<Hash, uint64_t> chunk_offsets_;
+
+  // Current server file offset.
+  uint64_t curr_offset_ = 0;
+
+  // Whether all server files have been received.
+  bool all_chunks_received_ = false;
+};
+
+class PatchSender {
+  // 1 byte for source, 8 bytes for offset and 4 bytes for size.
+  static constexpr size_t kPatchMetadataSize =
+      sizeof(uint8_t) + sizeof(uint64_t) + sizeof(uint32_t);
+
+ public:
+  PatchSender(FILE* file, MessagePump* message_pump)
+      : file_(file), message_pump_(message_pump) {}
+
+  // Tries to send patch data for the next chunk in |client_chunks|. The class
+  // keeps an internal counter for the current chunk index. Patch data is not
+  // sent if the current client chunk is not found among the server chunks and
+  // there are outstanding server chunks. In that case, the method returns
+  // with an OK status and should be called later as soon as additional server
+  // chunks have been received.
+  // |num_client_bytes_processed| is set to the total size of the chunks added.
+  absl::Status TryAddChunks(const std::vector<Chunk>& client_chunks,
+                            const ServerChunkReceiver& server_chunk_receiver,
+                            uint64_t* num_client_bytes_processed) {
+    assert(num_client_bytes_processed);
+    *num_client_bytes_processed = 0;
+
+    while (curr_chunk_idx_ < client_chunks.size()) {
+      const Chunk& chunk = client_chunks[curr_chunk_idx_];
+      auto it = server_chunk_receiver.ChunkOffsets().find(chunk.hash);
+      bool exists = it != server_chunk_receiver.ChunkOffsets().end();
+
+      // If there are outstanding server chunks and the client hash is not
+      // found, do not send the patch data yet. A future server chunk might
+      // contain the data.
+      if (!exists && !server_chunk_receiver.AllChunksReceived()) {
+        return absl::OkStatus();
+      }
+
+      absl::Status status = exists ? AddExistingChunk(it->second, chunk.size)
+                                   : AddNewChunk(chunk.size);
+      if (!status.ok()) {
+        return WrapStatus(status, "Failed to add chunk");
+      }
+
+      ++curr_chunk_idx_;
+      *num_client_bytes_processed += chunk.size;
+
+      // Break loop if all server chunks are received. Otherwise, progress
+      // reporting is blocked.
+      if (server_chunk_receiver.AllChunksReceived()) {
+        break;
+      }
+    }
+
+    return absl::OkStatus();
+  }
+
+  // Sends the remaining patch commands and an EOF marker.
+  absl::Status Flush() {
+    if (request_size_ > 0) {
+      absl::Status status =
+          message_pump_->SendMessage(PacketType::kAddPatchCommands, request_);
+      if (!status.ok()) {
+        return WrapStatus(status, "Failed to send final patch commands");
+      }
+      total_request_size_ += request_size_;
+      request_.Clear();
+    }
+
+    // Send an empty patch commands request as EOF marker.
+    absl::Status status =
+        message_pump_->SendMessage(PacketType::kAddPatchCommands, request_);
+    if (!status.ok()) {
+      return WrapStatus(status, "Failed to send patch commands EOF marker");
+    }
+
+    return absl::OkStatus();
+  }
+
+  // Returns the (estimated) total size of all patch data sent.
+  uint64_t GetTotalRequestSize() const { return total_request_size_; }
+
+  // Index of the next client chunk.
+  size_t CurrChunkIdx() const { return curr_chunk_idx_; }
+
+ private:
+  // Adds patch data for a client chunk that has a matching server chunk of
+  // given |size| at given |offset| in the server file.
+  absl::Status AddExistingChunk(uint64_t offset, uint32_t size) {
+    int last_idx = request_.sources_size() - 1;
+    if (last_idx >= 0 &&
+        request_.sources(last_idx) ==
+            AddPatchCommandsRequest::SOURCE_BASIS_FILE &&
+        request_.offsets(last_idx) + request_.sizes(last_idx) == offset &&
+        request_.sizes(last_idx) < kCombinedChunkSizeThreshold) {
+      // Same source and contiguous data -> Append to last entry.
+      request_.set_sizes(last_idx, request_.sizes(last_idx) + size);
+    } else {
+      // Different source or first chunk -> Create new entry.
+      request_.add_sources(AddPatchCommandsRequest::SOURCE_BASIS_FILE);
+      request_.add_offsets(offset);
+      request_.add_sizes(size);
+      request_size_ += kPatchMetadataSize;
+    }
+
+    return OnChunkAdded(size);
+  }
+
+  absl::Status AddNewChunk(uint32_t size) {
+    std::string* data = request_.mutable_data();
+    int last_idx = request_.sources_size() - 1;
+    if (last_idx >= 0 &&
+        request_.sources(last_idx) == AddPatchCommandsRequest::SOURCE_DATA) {
+      // Same source -> Append to last entry.
+      request_.set_sizes(last_idx, request_.sizes(last_idx) + size);
+    } else {
+      // Different source or first chunk -> Create new entry.
+      request_.add_sources(AddPatchCommandsRequest::SOURCE_DATA);
+      request_.add_offsets(data->size());
+      request_.add_sizes(size);
+      request_size_ += kPatchMetadataSize;
+    }
+
+    // Read data from client file into |data|. Be sure to restore the previous
+    // file offset as the chunker might still be processing the file.
+    size_t prev_size = data->size();
+    data->resize(prev_size + size);
+    int64_t prev_offset = ftell64(file_);
+    if (fseek64(file_, file_offset_, SEEK_SET) != 0 ||
+        fread(&(*data)[prev_size], 1, size, file_) != size ||
+        fseek64(file_, prev_offset, SEEK_SET) != 0) {
+      return MakeStatus("Failed to read %u bytes at offset %u", size,
+                        file_offset_);
+    }
+    request_size_ += size;
+
+    return OnChunkAdded(size);
+  }
+
+  absl::Status OnChunkAdded(uint32_t size) {
+    file_offset_ += size;
+
+    // Send patch commands if there's enough data.
+    if (request_size_ > kPatchRequestSizeThreshold) {
+      absl::Status status =
+          message_pump_->SendMessage(PacketType::kAddPatchCommands, request_);
+      if (!status.ok()) {
+        return WrapStatus(status, "Failed to send patch commands");
+      }
+      total_request_size_ += request_size_;
+      request_size_ = 0;
+      request_.Clear();
+    }
+
+    return absl::OkStatus();
+  }
+
+  FILE* file_;
+  MessagePump* message_pump_;
+
+  AddPatchCommandsRequest request_;
+  size_t request_size_ = 0;
+  size_t total_request_size_ = 0;
+  uint64_t file_offset_ = 0;
+  size_t curr_chunk_idx_ = 0;
+};
+
+}  // namespace
+
+CdcInterface::CdcInterface(MessagePump* message_pump)
+    : message_pump_(message_pump) {}
+
+absl::Status CdcInterface::CreateAndSendSignature(const std::string& filepath) {
+  absl::StatusOr<FILE*> file = path::OpenFile(filepath, "rb");
+  if (!file.ok()) {
+    return file.status();
+  }
+#if PLATFORM_LINUX
+  // Tell the kernel we'll load the file sequentially (improves IO bandwidth).
+  posix_fadvise(fileno(*file), 0, 0, POSIX_FADV_SEQUENTIAL);
+#endif
+
+  // Use a background thread for computing hashes on the server.
+  // Allocate lazily since it is not needed on the client.
+  // MUST NOT use more than 1 worker thread since the order of finished tasks
+  // would then not necessarily match the pushing order. However, the order is
+  // important for computing offsets.
+  if (!hash_pool_) hash_pool_ = std::make_unique<Threadpool>(1);
+
+  // |chunk_handler| is called for each CDC chunk. It pushes a hash task to the
+  // pool. Tasks are "recycled" from |free_tasks_|, so that buffers don't have
+  // to reallocated constantly.
+  size_t num_hash_tasks = 0;
+  auto chunk_handler = [pool = hash_pool_.get(), &num_hash_tasks,
+                        free_tasks = &free_tasks_](const void* data,
+                                                   size_t size) {
+    ++num_hash_tasks;
+    if (free_tasks->empty()) {
+      free_tasks->push_back(std::make_unique<HashTask>());
+    }
+    std::unique_ptr<Task> task = std::move(free_tasks->back());
+    free_tasks->pop_back();
+    static_cast<HashTask*>(task.get())->SetData(data, size);
+    pool->QueueTask(std::move(task));
+  };
+
+  fastcdc::Config config(kMinChunkSize, kAvgChunkSize, kMaxChunkSize);
+  fastcdc::Chunker chunker(config, chunk_handler);
+
+  AddSignaturesResponse response;
+  auto read_handler = [&chunker, &response, pool = hash_pool_.get(),
+                       &num_hash_tasks, free_tasks = &free_tasks_,
+                       message_pump = message_pump_](const void* data,
+                                                     size_t size) {
+    chunker.Process(static_cast<const uint8_t*>(data), size);
+
+    // Finish hashing tasks. Block if there are too many of them in flight.
+    for (;;) {
+      std::unique_ptr<Task> task = num_hash_tasks >= kMaxNumHashTasks
+                                       ? pool->GetCompletedTask()
+                                       : pool->TryGetCompletedTask();
+      if (!task) break;
+      num_hash_tasks--;
+      static_cast<HashTask*>(task.get())->AppendHash(&response);
+      free_tasks->push_back(std::move(task));
+    }
+
+    // Send data if we have enough chunks.
+    if (response.sizes_size() >= kMinNumChunksPerBatch) {
+      absl::Status status =
+          message_pump->SendMessage(PacketType::kAddSignatures, response);
+      if (!status.ok()) {
+        return WrapStatus(status, "Failed to send signatures");
+      }
+      response.Clear();
+    }
+
+    return absl::OkStatus();
+  };
+
+  absl::Status status =
+      path::StreamReadFileContents(*file, kFileIoBufferSize, read_handler);
+  fclose(*file);
+  if (!status.ok()) {
+    return WrapStatus(status, "Failed to compute signatures");
+  }
+  chunker.Finalize();
+
+  // Finish hashing tasks.
+  hash_pool_->Wait();
+  std::unique_ptr<Task> task = hash_pool_->TryGetCompletedTask();
+  while (task) {
+    static_cast<HashTask*>(task.get())->AppendHash(&response);
+    free_tasks_.push_back(std::move(task));
+    task = hash_pool_->TryGetCompletedTask();
+  }
+
+  // Send the remaining chunks, if any.
+  if (response.sizes_size() > 0) {
+    status = message_pump_->SendMessage(PacketType::kAddSignatures, response);
+    if (!status.ok()) {
+      return WrapStatus(status, "Failed to send final signatures");
+    }
+    response.Clear();
+  }
+
+  // Send an empty response as EOF marker.
+  status = message_pump_->SendMessage(PacketType::kAddSignatures, response);
+  if (!status.ok()) {
+    return WrapStatus(status, "Failed to send signatures EOF marker");
+  }
+
+  return absl::OkStatus();
+}
+
+absl::Status CdcInterface::ReceiveSignatureAndCreateAndSendDiff(
+    FILE* file, ReportCdcProgress* progress) {
+  //
+  // Compute signatures from client |file| and send patches while receiving
+  // server signatures.
+  //
+  std::vector<Chunk> client_chunks;
+  ServerChunkReceiver server_chunk_receiver(message_pump_);
+  PatchSender patch_sender(file, message_pump_);
+
+  auto chunk_handler = [&client_chunks](const void* data, size_t size) {
+    client_chunks.emplace_back(ComputeHash(data, size),
+                               static_cast<uint32_t>(size));
+  };
+
+  fastcdc::Config config(kMinChunkSize, kAvgChunkSize, kMaxChunkSize);
+  fastcdc::Chunker chunker(config, chunk_handler);
+
+  uint64_t file_size = 0;
+  auto read_handler = [&chunker, &client_chunks, &server_chunk_receiver,
+                       &file_size, progress,
+                       &patch_sender](const void* data, size_t size) {
+    // Process client chunks for the data read.
+    chunker.Process(static_cast<const uint8_t*>(data), size);
+    file_size += size;
+
+    const bool all_client_chunks_read = data == nullptr;
+    if (all_client_chunks_read) {
+      chunker.Finalize();
+    }
+
+    do {
+      // Receive any server chunks available.
+      uint64_t num_server_bytes_processed = 0;
+      absl::Status status = server_chunk_receiver.Receive(
+          /*block=*/all_client_chunks_read, &num_server_bytes_processed);
+      if (!status.ok()) {
+        return WrapStatus(status, "Failed to receive server chunks");
+      }
+
+      // Try to send patch data.
+      uint64_t num_client_bytes_processed = 0;
+      status = patch_sender.TryAddChunks(client_chunks, server_chunk_receiver,
+                                         &num_client_bytes_processed);
+      if (!status.ok()) {
+        return WrapStatus(status, "Failed to send patch data");
+      }
+
+      progress->ReportSyncProgress(num_client_bytes_processed,
+                                   num_server_bytes_processed);
+    } while (all_client_chunks_read &&
+             (!server_chunk_receiver.AllChunksReceived() ||
+              patch_sender.CurrChunkIdx() < client_chunks.size()));
+
+    return absl::OkStatus();
+  };
+
+  absl::Status status =
+      path::StreamReadFileContents(file, kFileIoBufferSize, read_handler);
+  if (!status.ok()) {
+    return WrapStatus(status, "Failed to stream file");
+  }
+
+  // Should have sent all client chunks by now.
+  assert(patch_sender.CurrChunkIdx() == client_chunks.size());
+
+  // Flush remaining patches.
+  status = patch_sender.Flush();
+  if (!status.ok()) {
+    return WrapStatus(status, "Failed to flush patches");
+  }
+
+  return absl::OkStatus();
+}
+
+absl::Status CdcInterface::ReceiveDiffAndPatch(
+    const std::string& basis_filepath, FILE* patched_file,
+    bool* is_executable) {
+  Buffer buffer;
+  *is_executable = false;
+
+  absl::StatusOr<FILE*> basis_file = path::OpenFile(basis_filepath, "rb");
+  if (!basis_file.ok()) {
+    return basis_file.status();
+  }
+#if PLATFORM_LINUX
+  // Tell the kernel we'll load the file sequentially (improves IO bandwidth).
+  // It is not strictly true that the basis file is accessed sequentially, but
+  // for larger parts of this file this should be the case.
+  posix_fadvise(fileno(*basis_file), 0, 0, POSIX_FADV_SEQUENTIAL);
+#endif
+
+  bool first_chunk = true;
+  for (;;) {
+    AddPatchCommandsRequest request;
+    absl::Status status =
+        message_pump_->ReceiveMessage(PacketType::kAddPatchCommands, &request);
+    if (!status.ok()) {
+      fclose(*basis_file);
+      return WrapStatus(status, "Failed to receive AddPatchCommandsRequest");
+    }
+
+    // All arrays must be of the same size.
+    int num_chunks = request.sources_size();
+    if (num_chunks != request.offsets_size() ||
+        num_chunks != request.sizes_size()) {
+      fclose(*basis_file);
+      return MakeStatus(
+          "Corrupted patch command arrays: Expected sizes %i. Actual %i/%i.",
+          num_chunks, request.offsets_size(), request.sizes_size());
+    }
+
+    if (num_chunks == 0) {
+      // A zero-size request marks the end of patch commands.
+      break;
+    }
+
+    for (int n = 0; n < num_chunks; ++n) {
+      AddPatchCommandsRequest::Source source = request.sources(n);
+      uint64_t chunk_offset = request.offsets(n);
+      uint32_t chunk_size = request.sizes(n);
+
+      const char* chunk_data = nullptr;
+      if (source == AddPatchCommandsRequest::SOURCE_BASIS_FILE) {
+        // Copy [chunk_offset, chunk_offset + chunk_size) from |basis_file|.
+        buffer.resize(chunk_size);
+        if (fseek64(*basis_file, chunk_offset, SEEK_SET) != 0 ||
+            fread(buffer.data(), 1, chunk_size, *basis_file) != chunk_size) {
+          fclose(*basis_file);
+          return MakeStatus(
+              "Failed to read %u bytes at offset %u from basis file",
+              chunk_size, chunk_offset);
+        }
+        chunk_data = buffer.data();
+      } else {
+        // Write [chunk_offset, chunk_offset + chunk_size) from request data.
+        assert(source == AddPatchCommandsRequest::SOURCE_DATA);
+        if (request.data().size() < chunk_offset + chunk_size) {
+          fclose(*basis_file);
+          return MakeStatus(
+              "Insufficient data in patch commands. Required %u. Actual %u.",
+              chunk_offset + chunk_size, request.data().size());
+        }
+        chunk_data = &request.data()[chunk_offset];
+      }
+
+      if (first_chunk && chunk_size > 0) {
+        first_chunk = false;
+        *is_executable = Util::IsExecutable(chunk_data, chunk_size);
+      }
+      if (fwrite(chunk_data, 1, chunk_size, patched_file) != chunk_size) {
+        fclose(*basis_file);
+        return MakeStatus("Failed to write %u bytes to patched file",
+                          chunk_size);
+      }
+    }
+  }
+  fclose(*basis_file);
+
+  return absl::OkStatus();
+}
+
+}  // namespace cdc_ft

cdc_rsync/base/cdc_interface.h

@@ -0,0 +1,73 @@
+/*
+ * Copyright 2022 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef CDC_RSYNC_BASE_CDC_INTERFACE_H_
+#define CDC_RSYNC_BASE_CDC_INTERFACE_H_
+
+#include <string>
+
+#include "absl/status/status.h"
+#include "common/threadpool.h"
+
+namespace cdc_ft {
+
+class MessagePump;
+
+class ReportCdcProgress {
+ public:
+  virtual ~ReportCdcProgress() = default;
+  virtual void ReportSyncProgress(size_t num_client_bytes_processed,
+                                  size_t num_server_bytes_processed) = 0;
+};
+
+// Creates signatures, diffs and patches files. Abstraction layer for fastcdc
+// chunking and blake3 hashing.
+class CdcInterface {
+ public:
+  explicit CdcInterface(MessagePump* message_pump);
+
+  // Creates the signature of the file at |filepath| and sends it to the socket.
+  // Typically called on the server.
+  absl::Status CreateAndSendSignature(const std::string& filepath);
+
+  // Receives the server-side signature of |file| from the socket, creates diff
+  // data using the signature and the file, and sends the diffs to the socket.
+  // Typically called on the client.
+  absl::Status ReceiveSignatureAndCreateAndSendDiff(
+      FILE* file, ReportCdcProgress* progress);
+
+  // Receives diffs from the socket and patches the file at |basis_filepath|.
+  // The patched data is written to |patched_file|, which must be open in "wb"
+  // mode. Sets |is_executable| to true if the patched file is an executable
+  // (based on magic headers).
+  // Typically called on the server.
+  absl::Status ReceiveDiffAndPatch(const std::string& basis_filepath,
+                                   FILE* patched_file, bool* is_executable);
+
+ private:
+  MessagePump* const message_pump_;
+
+  // Thread pool for computing chunk hashes.
+  std::unique_ptr<Threadpool> hash_pool_;
+
+  // List of unused hash computation tasks. Tasks are reused by the hash pool
+  // in order to prevent buffer reallocation.
+  std::vector<std::unique_ptr<Task>> free_tasks_;
+};
+
+}  // namespace cdc_ft
+
+#endif  // CDC_RSYNC_BASE_CDC_INTERFACE_H_

cdc_rsync/base/cdc_interface_test.cc

@@ -0,0 +1,118 @@
+// Copyright 2022 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "cdc_rsync/base/cdc_interface.h"
+
+#include <cstdio>
+#include <fstream>
+
+#include "cdc_rsync/base/fake_socket.h"
+#include "cdc_rsync/base/message_pump.h"
+#include "common/log.h"
+#include "common/path.h"
+#include "common/status_test_macros.h"
+#include "common/test_main.h"
+#include "gtest/gtest.h"
+
+namespace cdc_ft {
+namespace {
+
+class FakeCdcProgress : public ReportCdcProgress {
+ public:
+  void ReportSyncProgress(uint64_t num_client_bytes_processed,
+                          uint64_t num_server_bytes_processed) override {
+    total_client_bytes_processed += num_client_bytes_processed;
+    total_server_bytes_processed += num_server_bytes_processed;
+  }
+
+  uint64_t total_client_bytes_processed = 0;
+  uint64_t total_server_bytes_processed = 0;
+};
+
+class CdcInterfaceTest : public ::testing::Test {
+ public:
+  void SetUp() override {
+    Log::Initialize(std::make_unique<ConsoleLog>(LogLevel::kInfo));
+    message_pump_.StartMessagePump();
+  }
+
+  void TearDown() override {
+    socket_.ShutdownSendingEnd();
+    message_pump_.StopMessagePump();
+    Log::Shutdown();
+  }
+
+ protected:
+  FakeSocket socket_;
+  MessagePump message_pump_{&socket_, MessagePump::PacketReceivedDelegate()};
+
+  std::string base_dir_ = GetTestDataDir("cdc_interface");
+};
+
+TEST_F(CdcInterfaceTest, SyncTest) {
+  CdcInterface cdc(&message_pump_);
+  FakeCdcProgress progress;
+
+  const std::string old_filepath = path::Join(base_dir_, "old_file.txt");
+  const std::string new_filepath = path::Join(base_dir_, "new_file.txt");
+  const std::string patched_filepath =
+      path::Join(base_dir_, "patched_file.txt");
+
+  path::Stats old_stats;
+  EXPECT_OK(path::GetStats(old_filepath, &old_stats));
+
+  path::Stats new_stats;
+  EXPECT_OK(path::GetStats(new_filepath, &new_stats));
+
+  // Create signature of old file and send it to the fake socket (it'll just
+  // send it to itself).
+  EXPECT_OK(cdc.CreateAndSendSignature(old_filepath));
+
+  // Receive the signature from the fake socket, generate the diff to the file
+  // at |new_filepath| and send it to the socket again.
+  absl::StatusOr<FILE*> new_file = path::OpenFile(new_filepath, "rb");
+  EXPECT_OK(new_file);
+  EXPECT_OK(cdc.ReceiveSignatureAndCreateAndSendDiff(*new_file, &progress));
+  fclose(*new_file);
+
+  // Receive the diff from the fake socket and create a patched file.
+  std::FILE* patched_file = std::tmpfile();
+  ASSERT_TRUE(patched_file != nullptr);
+  bool is_executable = false;
+  EXPECT_OK(
+      cdc.ReceiveDiffAndPatch(old_filepath, patched_file, &is_executable));
+  EXPECT_FALSE(is_executable);
+
+  // Read new file.
+  std::ifstream new_file_stream(new_filepath.c_str(), std::ios::binary);
+  std::vector<uint8_t> new_file_data(
+      std::istreambuf_iterator<char>(new_file_stream), {});
+
+  // Read patched file.
+  fseek(patched_file, 0, SEEK_END);
+  std::vector<uint8_t> patched_file_data(ftell(patched_file));
+  fseek(patched_file, 0, SEEK_SET);
+  fread(patched_file_data.data(), 1, patched_file_data.size(), patched_file);
+
+  // New and patched file should be equal now.
+  EXPECT_EQ(patched_file_data, new_file_data);
+  fclose(patched_file);
+
+  // Verify progress tracker.
+  EXPECT_EQ(progress.total_server_bytes_processed, old_stats.size);
+  EXPECT_EQ(progress.total_client_bytes_processed, new_stats.size);
+}
+
+}  // namespace
+}  // namespace cdc_ft

cdc_rsync/base/fake_socket.cc

@@ -0,0 +1,70 @@
+// Copyright 2022 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "cdc_rsync/base/fake_socket.h"
+
+namespace cdc_ft {
+
+FakeSocket::FakeSocket() = default;
+
+FakeSocket::~FakeSocket() = default;
+
+absl::Status FakeSocket::Send(const void* buffer, size_t size) {
+  // Wait until we can send again.
+  std::unique_lock<std::mutex> suspend_lock(suspend_mutex_);
+  suspend_cv_.wait(suspend_lock, [this]() { return !sending_suspended_; });
+  suspend_lock.unlock();
+
+  std::unique_lock<std::mutex> lock(data_mutex_);
+  data_.append(static_cast<const char*>(buffer), size);
+  lock.unlock();
+  data_cv_.notify_all();
+  return absl::OkStatus();
+}
+
+absl::Status FakeSocket::Receive(void* buffer, size_t size,
+                                 bool allow_partial_read,
+                                 size_t* bytes_received) {
+  *bytes_received = 0;
+  std::unique_lock<std::mutex> lock(data_mutex_);
+  data_cv_.wait(lock, [this, size, allow_partial_read]() {
+    return allow_partial_read || data_.size() >= size || shutdown_;
+  });
+  if (shutdown_) {
+    return absl::UnavailableError("Pipe is shut down");
+  }
+  size_t to_copy = std::min(size, data_.size());
+  memcpy(buffer, data_.data(), to_copy);
+  *bytes_received = to_copy;
+
+  // This is horribly inefficent, but should be OK in a fake.
+  data_.erase(0, to_copy);
+  return absl::OkStatus();
+}
+
+void FakeSocket::ShutdownSendingEnd() {
+  std::unique_lock<std::mutex> lock(data_mutex_);
+  shutdown_ = true;
+  lock.unlock();
+  data_cv_.notify_all();
+}
+
+void FakeSocket::SuspendSending(bool suspended) {
+  std::unique_lock<std::mutex> lock(suspend_mutex_);
+  sending_suspended_ = suspended;
+  lock.unlock();
+  suspend_cv_.notify_all();
+}
+
+}  // namespace cdc_ft

cdc_rsync/base/fake_socket.h

@@ -0,0 +1,57 @@
+/*
+ * Copyright 2022 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef CDC_RSYNC_BASE_FAKE_SOCKET_H_
+#define CDC_RSYNC_BASE_FAKE_SOCKET_H_
+
+#include <condition_variable>
+#include <mutex>
+
+#include "absl/status/status.h"
+#include "cdc_rsync/base/socket.h"
+
+namespace cdc_ft {
+
+// Fake socket that receives the same data it sends.
+class FakeSocket : public Socket {
+ public:
+  FakeSocket();
+  ~FakeSocket();
+
+  // Socket:
+  absl::Status Send(const void* buffer, size_t size) override;  // thread-safe
+  absl::Status Receive(void* buffer, size_t size, bool allow_partial_read,
+                       size_t* bytes_received) override;  // thread-safe
+
+  void ShutdownSendingEnd();
+
+  // If set to true, blocks on Send() until it is set to false again.
+  void SuspendSending(bool suspended);
+
+ private:
+  std::mutex data_mutex_;
+  std::condition_variable data_cv_;
+  std::string data_;
+  bool shutdown_ = false;
+
+  bool sending_suspended_ = false;
+  std::mutex suspend_mutex_;
+  std::condition_variable suspend_cv_;
+};
+
+}  // namespace cdc_ft
+
+#endif  // CDC_RSYNC_BASE_FAKE_SOCKET_H_

cdc_rsync/base/message_pump.cc

@@ -0,0 +1,473 @@
+// Copyright 2022 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "cdc_rsync/base/message_pump.h"
+
+#include "absl/status/status.h"
+#include "absl/strings/str_format.h"
+#include "cdc_rsync/base/socket.h"
+#include "common/buffer.h"
+#include "common/log.h"
+#include "common/status.h"
+#include "google/protobuf/message_lite.h"
+
+namespace cdc_ft {
+namespace {
+
+// Max total size of messages in the packet queues.
+// If exdeeded, Send/Receive methods start blocking.
+uint64_t kInOutBufferSize = 1024 * 1024 * 8;
+
+// Header is 1 byte type, 3 bytes size.
+constexpr size_t kHeaderSize = 4;
+
+// Size is compressed to 3 bytes.
+constexpr uint32_t kMaxPacketSize = 256 * 256 * 256 - 1;
+
+// Creates a packet of size |kHeaderSize| + |size| and sets the header.
+absl::Status CreateSerializedPacket(PacketType type, size_t size,
+                                    Buffer* serialized_packet) {
+  if (size > kMaxPacketSize) {
+    return MakeStatus("Max packet size exceeded: %u", size);
+  }
+
+  serialized_packet->clear();
+  serialized_packet->reserve(kHeaderSize + size);
+
+  // Header is 1 byte type, 3 bytes size.
+  static_assert(static_cast<size_t>(PacketType::kCount) <= 256, "");
+  static_assert(kMaxPacketSize < 256 * 256 * 256, "");
+  static_assert(kHeaderSize == 4, "");
+
+  uint8_t header[] = {static_cast<uint8_t>(type),
+                      static_cast<uint8_t>(size & 0xFF),
+                      static_cast<uint8_t>((size >> 8) & 0xFF),
+                      static_cast<uint8_t>((size >> 16) & 0xFF)};
+  serialized_packet->append(header, sizeof(header));
+  return absl::OkStatus();
+}
+
+#define HANDLE_PACKET_TYPE(type) \
+  case PacketType::type:         \
+    return #type;
+
+const char* PacketTypeName(PacketType type) {
+  if (type > PacketType::kCount) {
+    return "<unknown>";
+  }
+
+  switch (type) {
+    HANDLE_PACKET_TYPE(kRawData)
+    HANDLE_PACKET_TYPE(kTest)
+    HANDLE_PACKET_TYPE(kSetOptions)
+    HANDLE_PACKET_TYPE(kToggleCompression)
+    HANDLE_PACKET_TYPE(kAddFiles)
+    HANDLE_PACKET_TYPE(kSendFileStats)
+    HANDLE_PACKET_TYPE(kAddFileIndices)
+    HANDLE_PACKET_TYPE(kSendMissingFileData)
+    HANDLE_PACKET_TYPE(kAddSignatures)
+    HANDLE_PACKET_TYPE(kAddPatchCommands)
+    HANDLE_PACKET_TYPE(kAddDeletedFiles)
+    HANDLE_PACKET_TYPE(kShutdown)
+    HANDLE_PACKET_TYPE(kCount)
+  }
+
+  return "<unknown>";
+}
+
+#undef HANDLE_PACKET_TYPE
+
+}  // namespace
+
+MessagePump::MessagePump(Socket* socket, PacketReceivedDelegate packet_received)
+    : socket_(socket),
+      packet_received_(packet_received),
+      creation_thread_id_(std::this_thread::get_id()) {
+  assert(socket_ != nullptr);
+}
+
+MessagePump::~MessagePump() { StopMessagePump(); }
+
+void MessagePump::StartMessagePump() {
+  assert(creation_thread_id_ == std::this_thread::get_id());
+
+  message_sender_thread_ = std::thread([this]() { ThreadSenderMain(); });
+  message_receiver_thread_ = std::thread([this]() { ThreadReceiverMain(); });
+}
+
+void MessagePump::StopMessagePump() {
+  assert(creation_thread_id_ == std::this_thread::get_id());
+
+  if (shutdown_) {
+    return;
+  }
+
+  FlushOutgoingQueue();
+
+  {
+    absl::MutexLock outgoing_lock(&outgoing_mutex_);
+    absl::MutexLock incoming_lock(&incoming_mutex_);
+    shutdown_ = true;
+  }
+
+  if (message_sender_thread_.joinable()) {
+    message_sender_thread_.join();
+  }
+
+  if (message_receiver_thread_.joinable()) {
+    message_receiver_thread_.join();
+  }
+}
+
+absl::Status MessagePump::SendRawData(const void* data, size_t size) {
+  Buffer serialized_packet;
+  absl::Status status =
+      CreateSerializedPacket(PacketType::kRawData, size, &serialized_packet);
+  if (!status.ok()) {
+    return status;
+  }
+  const uint8_t* u8_data = static_cast<const uint8_t*>(data);
+  serialized_packet.append(u8_data, size);
+  return QueuePacket(std::move(serialized_packet));
+}
+
+absl::Status MessagePump::SendMessage(
+    PacketType type, const google::protobuf::MessageLite& message) {
+  Buffer serialized_packet;
+  size_t size = message.ByteSizeLong();
+  absl::Status status = CreateSerializedPacket(type, size, &serialized_packet);
+  if (!status.ok()) {
+    return status;
+  }
+
+  // Serialize the message directly into the packet.
+  serialized_packet.resize(kHeaderSize + size);
+  if (size > 0 &&
+      !message.SerializeToArray(serialized_packet.data() + kHeaderSize,
+                                static_cast<int>(size))) {
+    return MakeStatus("Failed to serialize message to array");
+  }
+
+  return QueuePacket(std::move(serialized_packet));
+}
+
+absl::Status MessagePump::QueuePacket(Buffer&& serialize_packet) {
+  // Wait a little if the max queue size is exceeded.
+  absl::MutexLock outgoing_lock(&outgoing_mutex_);
+  auto cond = [this]() ABSL_EXCLUSIVE_LOCKS_REQUIRED(outgoing_mutex_) {
+    return outgoing_packets_byte_size_ < kInOutBufferSize || send_error_ ||
+           receive_error_;
+  };
+  outgoing_mutex_.Await(absl::Condition(&cond));
+
+  // There could be a race where send_error_ is set to true after this, but
+  // that's OK.
+  if (send_error_ || receive_error_) {
+    absl::MutexLock status_lock(&status_mutex_);
+    return WrapStatus(status_,
+                      "Failed to send packet. Message pump thread is down");
+  }
+
+  // Put packet into outgoing queue.
+  outgoing_packets_byte_size_ += serialize_packet.size();
+  outgoing_packets_.push(std::move(serialize_packet));
+
+  return absl::OkStatus();
+}
+
+absl::Status MessagePump::ThreadDoSendPacket(Buffer&& serialized_packet) {
+  if (receive_error_) {
+    // Just eat the packet if there was a receive error as the other side is
+    // probably down and won't read packets anymore.
+    return absl::OkStatus();
+  }
+
+  if (output_handler_) {
+    // Redirect output, don't send to socket.
+    absl::Status status =
+        output_handler_(serialized_packet.data(), serialized_packet.size());
+    return WrapStatus(status, "Output handler failed");
+  }
+
+  absl::Status status =
+      socket_->Send(serialized_packet.data(), serialized_packet.size());
+  if (!status.ok()) {
+    return WrapStatus(status, "Failed to send packet of size %u",
+                      serialized_packet.size());
+  }
+
+  LOG_VERBOSE("Sent packet of size %u (total buffer: %u)",
+              serialized_packet.size(), outgoing_packets_byte_size_.load());
+
+  return absl::OkStatus();
+}
+
+absl::Status MessagePump::ReceiveRawData(Buffer* data) {
+  Packet packet;
+  absl::Status status = DequeuePacket(&packet);
+  if (!status.ok()) {
+    return WrapStatus(status, "Failed to dequeue packet");
+  }
+
+  if (packet.type != PacketType::kRawData) {
+    return MakeStatus("Unexpected packet type %s. Expected kRawData.",
+                      PacketTypeName(packet.type));
+  }
+
+  *data = std::move(packet.data);
+  return absl::OkStatus();
+}
+
+absl::Status MessagePump::ReceiveMessage(
+    PacketType type, google::protobuf::MessageLite* message) {
+  Packet packet;
+  absl::Status status = DequeuePacket(&packet);
+  if (!status.ok()) {
+    return WrapStatus(status, "Failed to dequeue packet");
+  }
+
+  if (packet.type != type) {
+    return MakeStatus("Unexpected packet type %s. Expected %s.",
+                      PacketTypeName(packet.type), PacketTypeName(type));
+  }
+
+  if (!message->ParseFromArray(packet.data.data(),
+                               static_cast<int>(packet.data.size()))) {
+    return MakeStatus("Failed to parse packet of type %s and size %u",
+                      PacketTypeName(packet.type), packet.data.size());
+  }
+  return absl::OkStatus();
+}
+
+absl::Status MessagePump::DequeuePacket(Packet* packet) {
+  // Wait for a packet to be available.
+  absl::MutexLock incoming_lock(&incoming_mutex_);
+  auto cond = [this]() ABSL_EXCLUSIVE_LOCKS_REQUIRED(incoming_mutex_) {
+    return !incoming_packets_.empty() || send_error_ || receive_error_;
+  };
+  incoming_mutex_.Await(absl::Condition(&cond));
+
+  // If receive_error_ is true, do not return an error until |incoming_packets_|
+  // is empty and all valid packets have been returned. This way, the error
+  // shows up for the packet that failed to be received.
+  if (send_error_ || (receive_error_ && incoming_packets_.empty())) {
+    absl::MutexLock status_lock(&status_mutex_);
+    return WrapStatus(status_, "Message pump thread is down");
+  }
+
+  // Grab packet from incoming queue.
+  *packet = std::move(incoming_packets_.front());
+  incoming_packets_.pop();
+
+  // Update byte size.
+  incoming_packets_byte_size_ -= kHeaderSize + packet->data.size();
+
+  return absl::OkStatus();
+}
+
+absl::Status MessagePump::ThreadDoReceivePacket(Packet* packet) {
+  // Read type and size in one go for performance reasons.
+  uint8_t header[kHeaderSize];
+  absl::Status status = ThreadDoReceive(&header, kHeaderSize);
+  if (!status.ok()) {
+    return WrapStatus(status, "Failed to receive packet of size %u",
+                      kHeaderSize);
+  }
+
+  static_assert(kHeaderSize == 4, "");
+
+  uint8_t packet_type = header[0];
+  uint32_t packet_size = static_cast<uint32_t>(header[1]) |
+                         (static_cast<uint32_t>(header[2]) << 8) |
+                         (static_cast<uint32_t>(header[3]) << 16);
+
+  if (packet_type >= static_cast<uint8_t>(PacketType::kCount)) {
+    return MakeStatus("Invalid packet type: %u", packet_type);
+  }
+  packet->type = static_cast<PacketType>(packet_type);
+
+  if (packet_size > kMaxPacketSize) {
+    return MakeStatus("Max packet size exceeded: %u", packet_size);
+  }
+
+  packet->data.resize(packet_size);
+  status = ThreadDoReceive(packet->data.data(), packet_size);
+  if (!status.ok()) {
+    return WrapStatus(status, "Failed to read packet data of size %u",
+                      packet_size);
+  }
+
+  LOG_VERBOSE("Received packet of size %u (total buffer: %u)", packet_size,
+              incoming_packets_byte_size_.load());
+
+  return absl::OkStatus();
+}
+
+absl::Status MessagePump::ThreadDoReceive(void* buffer, size_t size) {
+  if (size == 0) {
+    return absl::OkStatus();
+  }
+
+  if (input_reader_) {
+    size_t bytes_read = 0;
+    bool eof = false;
+    absl::Status status = input_reader_->Read(buffer, size, &bytes_read, &eof);
+    if (eof) {
+      input_reader_.reset();
+    }
+    if (!status.ok()) {
+      return status;
+    }
+
+    // |input_reader_| should read |size| bytes unless |eof| is hit.
+    assert(bytes_read == size || eof);
+
+    // Since this method never reads across packet boundaries and since packets
+    // should not be partially received through |input_reader_|, it is an error
+    // if there's a partial read on EOF.
+    if (eof && (bytes_read > 0 && bytes_read < size)) {
+      return MakeStatus("EOF after partial read of %u / %u bytes", bytes_read,
+                        size);
+    }
+
+    // Special case, might happen if |input_reader_| was an unzip stream and the
+    // last read stopped right before zlib's EOF marker. Fall through to reading
+    // uncompressed data in that case.
+    if (bytes_read == size) {
+      return absl::OkStatus();
+    }
+
+    assert(eof && bytes_read == 0);
+  }
+
+  size_t unused;
+  return socket_->Receive(buffer, size, /*allow_partial_read=*/false, &unused);
+}
+
+void MessagePump::FlushOutgoingQueue() {
+  absl::MutexLock outgoing_lock(&outgoing_mutex_);
+  auto cond = [this]() ABSL_EXCLUSIVE_LOCKS_REQUIRED(outgoing_mutex_) {
+    return outgoing_packets_byte_size_ == 0 || send_error_ || receive_error_;
+  };
+  outgoing_mutex_.Await(absl::Condition(&cond));
+}
+
+void MessagePump::RedirectInput(std::unique_ptr<InputReader> input_reader) {
+  assert(std::this_thread::get_id() == message_receiver_thread_.get_id());
+  assert(input_reader);
+
+  if (input_reader_) {
+    LOG_WARNING("Input reader already set");
+    return;
+  }
+
+  input_reader_ = std::move(input_reader);
+}
+
+void MessagePump::RedirectOutput(OutputHandler output_handler) {
+  FlushOutgoingQueue();
+  output_handler_ = std::move(output_handler);
+}
+
+size_t MessagePump::GetNumOutgoingPackagesForTesting() {
+  absl::MutexLock outgoing_lock(&outgoing_mutex_);
+  return outgoing_packets_.size();
+}
+
+size_t MessagePump::GetMaxInOutBufferSizeForTesting() {
+  return kInOutBufferSize;
+}
+
+size_t MessagePump::GetMaxPacketSizeForTesting() { return kMaxPacketSize; }
+
+void MessagePump::ThreadSenderMain() {
+  while (!send_error_) {
+    Buffer serialized_packet;
+    size_t size;
+    {
+      // Wait for a packet to be available.
+      absl::MutexLock outgoing_lock(&outgoing_mutex_);
+      auto cond = [this]() ABSL_EXCLUSIVE_LOCKS_REQUIRED(outgoing_mutex_) {
+        return outgoing_packets_.size() > 0 || shutdown_;
+      };
+      outgoing_mutex_.Await(absl::Condition(&cond));
+      if (shutdown_) {
+        break;
+      }
+
+      // Grab packet from outgoing queue.
+      serialized_packet = std::move(outgoing_packets_.front());
+      size = serialized_packet.size();
+      outgoing_packets_.pop();
+    }
+
+    // Send data. This blocks until all data is submitted.
+    absl::Status status = ThreadDoSendPacket(std::move(serialized_packet));
+    if (!status.ok()) {
+      {
+        absl::MutexLock status_lock(&status_mutex_);
+        status_ = WrapStatus(status, "Failed to send packet");
+      }
+      absl::MutexLock outgoing_lock(&outgoing_mutex_);
+      absl::MutexLock incoming_lock(&incoming_mutex_);
+      send_error_ = true;
+      break;
+    }
+
+    // Decrease AFTER sending, this is important for FlushOutgoingQueue().
+    absl::MutexLock outgoing_lock(&outgoing_mutex_);
+    outgoing_packets_byte_size_ -= size;
+  }
+}
+
+void MessagePump::ThreadReceiverMain() {
+  while (!receive_error_) {
+    // Wait for a packet to be available.
+    {
+      absl::MutexLock incoming_lock(&incoming_mutex_);
+      auto cond = [this]() ABSL_EXCLUSIVE_LOCKS_REQUIRED(incoming_mutex_) {
+        return incoming_packets_byte_size_ < kInOutBufferSize || shutdown_;
+      };
+      incoming_mutex_.Await(absl::Condition(&cond));
+      if (shutdown_) {
+        break;
+      }
+    }
+
+    // Receive packet. This blocks until data is available.
+    Packet packet;
+    absl::Status status = ThreadDoReceivePacket(&packet);
+    if (!status.ok()) {
+      {
+        absl::MutexLock status_lock(&status_mutex_);
+        status_ = WrapStatus(status, "Failed to receive packet");
+      }
+      absl::MutexLock outgoing_lock(&outgoing_mutex_);
+      absl::MutexLock incoming_lock(&incoming_mutex_);
+      receive_error_ = true;
+      break;
+    }
+
+    if (packet_received_) {
+      packet_received_(packet.type);
+    }
+
+    // Queue the packet for receiving.
+    absl::MutexLock incoming_lock(&incoming_mutex_);
+    incoming_packets_byte_size_ += kHeaderSize + packet.data.size();
+    incoming_packets_.push(std::move(packet));
+  }
+}
+
+}  // namespace cdc_ft

cdc_rsync/base/message_pump.h

@@ -0,0 +1,275 @@
+/*
+ * Copyright 2022 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef CDC_RSYNC_BASE_MESSAGE_PUMP_H_
+#define CDC_RSYNC_BASE_MESSAGE_PUMP_H_
+
+#include <queue>
+#include <thread>
+
+#include "absl/base/thread_annotations.h"
+#include "absl/status/status.h"
+#include "absl/synchronization/mutex.h"
+#include "common/buffer.h"
+
+namespace google {
+namespace protobuf {
+class MessageLite;
+}
+}  // namespace google
+
+namespace cdc_ft {
+
+class Socket;
+
+// See messages.proto. When sending a kXXXRequest from client to server or a
+// kXXXResponse from server to client, use packet type kXXX. See messages.proto.
+enum class PacketType {
+  // Not a proto, just raw bytes.
+  kRawData = 0,
+
+  // Used for testing.
+  kTest,
+
+  // Send options to server.
+  kSetOptions,
+
+  // Toggle compression on/off.
+  kToggleCompression,
+
+  //
+  // Send all files from client to server.
+  //
+
+  // Send file paths including timestamps and sizes, and directories to server.
+  // An empty request indicates that all data has been sent.
+  kAddFiles,
+  // Send stats about missing, excessive, changed and matching files to client.
+  kSendFileStats,
+
+  //
+  // Send all missing files from server to client.
+  //
+
+  // Send indices of missing files to client.
+  // An empty request indicates that all data has been sent.
+  // Also used for sending indices of changed files.
+  kAddFileIndices,
+
+  // Start sending missing file data to the server. After each
+  // SendMissingFileDataRequest, the client sends file data as raw packets and
+  // an empty packet to indicate eof.
+  kSendMissingFileData,
+
+  //
+  // Rsync data exchange.
+  //
+
+  // Send signatures to client.
+  // An empty response indicates that all data has been sent.
+  kAddSignatures,
+
+  // Send patch commands to server.
+  // An empty request indicates that all data has been sent.
+  kAddPatchCommands,
+
+  //
+  // Deletion of extraneous files.
+  //
+  kAddDeletedFiles,
+
+  //
+  // Shutdown.
+  //
+
+  // Ask the server to shut down. Also used for shutdown ack.
+  kShutdown,
+
+  // Must be last.
+  kCount
+};
+
+class MessagePump {
+ public:
+  using PacketReceivedDelegate = std::function<void(PacketType)>;
+
+  // |socket| is the underlying socket that data is sent to and received from,
+  // unless redirected with one of the Redirect* methods. |packet_received| is
+  // a callback that is called from the receiver thread as soon as a packet is
+  // received. RedirectInput() should be called from this delegate. Useful for
+  // things like decompression.
+  MessagePump(Socket* socket, PacketReceivedDelegate packet_received);
+  virtual ~MessagePump();
+
+  // Starts worker threads to send/receive messages. Should be called after the
+  // socket is connected. Must not be already started.
+  // NOT thread-safe. Should be called from the creation thread.
+  void StartMessagePump();
+
+  // Stops worker threads to send/receive messages. No-op if already stopped or
+  // not started. Cannot be restarted.
+  // NOT thread-safe. Should be called from the creation thread.
+  void StopMessagePump() ABSL_LOCKS_EXCLUDED(outgoing_mutex_, incoming_mutex_);
+
+  // Queues data for sending. May block if too much data is queued.
+  // Thread-safe.
+  absl::Status SendRawData(const void* data, size_t size);
+  absl::Status SendMessage(PacketType type,
+                           const google::protobuf::MessageLite& message);
+
+  // Receives a packet. Blocks if currently no packets is available.
+  // Thread-safe.
+  absl::Status ReceiveRawData(Buffer* data);
+  absl::Status ReceiveMessage(PacketType type,
+                              google::protobuf::MessageLite* message);
+
+  // Returns true if the Receive* functions have data available. Note that
+  // receiving messages from multiple threads might be racy, i.e. if
+  // CanReceive() returns true and Receive* is called afterwards, the method
+  // might block if another thread has grabbed the packet in the meantime.
+  bool CanReceive() const { return incoming_packets_byte_size_ > 0; }
+
+  // Blocks until all outgoing messages were sent. Does not prevent that other
+  // threads queue new packets while the method is blocking, so the caller
+  // should make sure that that's not the case for consistent behavior.
+  // Thread-safe.
+  void FlushOutgoingQueue() ABSL_LOCKS_EXCLUDED(outgoing_mutex_);
+
+  class InputReader {
+   public:
+    virtual ~InputReader() {}
+
+    // Reads as much as data possible to |out_buffer|, but no more than
+    // |out_size| bytes. Sets |bytes_read| to the number of bytes read.
+    // |eof| is set to true if no more input data is available. The flag
+    // indicates that the parent MessagePump should reset the input reader
+    // and read data from the socket again.
+    virtual absl::Status Read(void* out_buffer, size_t out_size,
+                              size_t* bytes_read, bool* eof) = 0;
+  };
+
+  // Starts receiving input from |input_reader| instead of from the socket.
+  // |input_reader| is called on a background thread. It must be a valid
+  // pointer. The input reader stays in place until it returns |eof| == true.
+  // After that, the input reader is reset and data is received from the socket
+  // again.
+  // This method must be called from the receiver thread, usually during the
+  // execution of the PacketReceivedDelegate passed in the constructor.
+  // Otherwise, the receiver thread might be blocked on a recv() call and the
+  // first data received would still be read the socket.
+  void RedirectInput(std::unique_ptr<InputReader> input_reader);
+
+  // If set to a non-empty function, starts sending output to |output_handler|
+  // instead of to the socket. If set to an empty function, starts sending to
+  // the socket again. |output_handler| is called on a background thread.
+  // The outgoing packet queue is flushed prior to changing the output handler.
+  // The caller must make sure that no background threads are sending new
+  // messages while this method is running.
+  using OutputHandler =
+      std::function<absl::Status(const void* data, size_t size)>;
+  void RedirectOutput(OutputHandler output_handler);
+
+  // Returns the number of packets queued for sending.
+  size_t GetNumOutgoingPackagesForTesting()
+      ABSL_LOCKS_EXCLUDED(outgoing_mutex_);
+
+  // Returns the max total size of messages in the packet queues.
+  size_t GetMaxInOutBufferSizeForTesting();
+
+  // Returns hte max size of a single raw or proto message (including header).
+  size_t GetMaxPacketSizeForTesting();
+
+ protected:
+  struct Packet {
+    PacketType type = PacketType::kCount;
+    Buffer data;
+
+    // Instances should be moved, not copied.
+    Packet() = default;
+    Packet(Packet&& other) { *this = std::move(other); }
+    Packet(const Packet&) = delete;
+    Packet& operator=(const Packet&) = delete;
+
+    Packet& operator=(Packet&& other) {
+      type = other.type;
+      data = std::move(other.data);
+      return *this;
+    }
+  };
+
+ private:
+  // Outgoing packets are already serialized to save mem copies.
+  absl::Status QueuePacket(Buffer&& serialized_packet)
+      ABSL_LOCKS_EXCLUDED(outgoing_mutex_, status_mutex_);
+  absl::Status DequeuePacket(Packet* packet)
+      ABSL_LOCKS_EXCLUDED(incoming_mutex_, status_mutex_);
+
+  // Underlying socket, not owned.
+  Socket* socket_;
+
+  // Delegate called if a packet was received.
+  // Called immediately from the receiver thread.
+  PacketReceivedDelegate packet_received_;
+
+  // Message pump threads main method for sending and receiving data.
+  void ThreadSenderMain() ABSL_LOCKS_EXCLUDED(outgoing_mutex_, status_mutex_);
+  void ThreadReceiverMain() ABSL_LOCKS_EXCLUDED(incoming_mutex_, status_mutex_);
+
+  // Actually send/receive packets.
+  absl::Status ThreadDoSendPacket(Buffer&& serialized_packet);
+  absl::Status ThreadDoReceivePacket(Packet* packet);
+  absl::Status ThreadDoReceive(void* buffer, size_t size);
+
+  std::thread message_sender_thread_;
+  std::thread message_receiver_thread_;
+
+  // If set, input is not received from the socket, but from |input_reader_|.
+  std::unique_ptr<InputReader> input_reader_;
+  // If set, output is not sent to the socket, but to |output_handler_|.
+  OutputHandler output_handler_;
+
+  //
+  // Synchronization of message pump threads and main thread.
+  //
+
+  // Guards to protect access to queued packets.
+  absl::Mutex outgoing_mutex_;
+  absl::Mutex incoming_mutex_ ABSL_ACQUIRED_AFTER(outgoing_mutex_);
+
+  // Queued packets.
+  std::queue<Buffer> outgoing_packets_ ABSL_GUARDED_BY(outgoing_mutex_);
+  std::queue<Packet> incoming_packets_ ABSL_GUARDED_BY(incoming_mutex_);
+
+  // Total size of queued packets. Used to limit max queue size.
+  std::atomic_uint64_t outgoing_packets_byte_size_{0};
+  std::atomic_uint64_t incoming_packets_byte_size_{0};
+
+  // If true, the respective thread saw an error and shut down.
+  std::atomic_bool send_error_{false};
+  std::atomic_bool receive_error_{false};
+
+  // Shutdown signal to sender and receiver threads.
+  std::atomic_bool shutdown_{false};
+
+  absl::Mutex status_mutex_;
+  absl::Status status_ ABSL_GUARDED_BY(status_mutex_);
+
+  std::thread::id creation_thread_id_;
+};
+
+}  // namespace cdc_ft
+
+#endif  // CDC_RSYNC_BASE_MESSAGE_PUMP_H_

cdc_rsync/base/message_pump_test.cc

@@ -0,0 +1,272 @@
+// Copyright 2022 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "cdc_rsync/base/message_pump.h"
+
+#include "cdc_rsync/base/fake_socket.h"
+#include "cdc_rsync/protos/messages.pb.h"
+#include "common/log.h"
+#include "common/status.h"
+#include "common/status_test_macros.h"
+#include "gtest/gtest.h"
+
+namespace cdc_ft {
+namespace {
+
+class MessagePumpTest : public ::testing::Test {
+ public:
+  void SetUp() override {
+    Log::Initialize(std::make_unique<ConsoleLog>(LogLevel::kInfo));
+    message_pump_.StartMessagePump();
+  }
+
+  void TearDown() override {
+    fake_socket_.ShutdownSendingEnd();
+    message_pump_.StopMessagePump();
+    Log::Shutdown();
+  }
+
+ protected:
+  // Called on the receiver thread.
+  void ThreadPackageReceived(PacketType type) {
+    // Empty by default. Only takes effect if set by tests.
+    if (type == PacketType::kToggleCompression) {
+      message_pump_.RedirectInput(std::move(fake_compressed_input_reader_));
+    }
+  }
+
+  FakeSocket fake_socket_;
+  MessagePump message_pump_{
+      &fake_socket_, [this](PacketType type) { ThreadPackageReceived(type); }};
+  std::unique_ptr<MessagePump::InputReader> fake_compressed_input_reader_;
+};
+
+TEST_F(MessagePumpTest, SendReceiveRawData) {
+  // The FakeSocket just routes everything that's sent to the receiving end.
+  const Buffer raw_data = {'r', 'a', 'w'};
+  EXPECT_OK(message_pump_.SendRawData(raw_data.data(), raw_data.size()));
+
+  Buffer received_raw_data;
+  EXPECT_OK(message_pump_.ReceiveRawData(&received_raw_data));
+
+  EXPECT_EQ(raw_data, received_raw_data);
+}
+
+TEST_F(MessagePumpTest, SendReceiveMessage) {
+  TestRequest request;
+  request.set_message("message");
+  EXPECT_OK(message_pump_.SendMessage(PacketType::kTest, request));
+
+  TestRequest received_request;
+  EXPECT_OK(message_pump_.ReceiveMessage(PacketType::kTest, &received_request));
+
+  EXPECT_EQ(request.message(), received_request.message());
+}
+
+TEST_F(MessagePumpTest, SendReceiveMultiple) {
+  const Buffer raw_data_1 = {'r', 'a', 'w', '1'};
+  const Buffer raw_data_2 = {'r', 'a', 'w', '2'};
+  TestRequest request;
+  request.set_message("message");
+
+  EXPECT_OK(message_pump_.SendRawData(raw_data_1.data(), raw_data_1.size()));
+  EXPECT_OK(message_pump_.SendMessage(PacketType::kTest, request));
+  EXPECT_OK(message_pump_.SendRawData(raw_data_2.data(), raw_data_2.size()));
+
+  Buffer received_raw_data_1;
+  Buffer received_raw_data_2;
+  TestRequest received_request;
+
+  EXPECT_OK(message_pump_.ReceiveRawData(&received_raw_data_1));
+  EXPECT_OK(message_pump_.ReceiveMessage(PacketType::kTest, &received_request));
+  EXPECT_OK(message_pump_.ReceiveRawData(&received_raw_data_2));
+
+  EXPECT_EQ(raw_data_1, received_raw_data_1);
+  EXPECT_EQ(request.message(), received_request.message());
+  EXPECT_EQ(raw_data_2, received_raw_data_2);
+}
+
+TEST_F(MessagePumpTest, ReceiveMessageInstreadOfRaw) {
+  const Buffer raw_data = {'r', 'a', 'w'};
+  EXPECT_OK(message_pump_.SendRawData(raw_data.data(), raw_data.size()));
+
+  TestRequest received_request;
+  EXPECT_NOT_OK(
+      message_pump_.ReceiveMessage(PacketType::kTest, &received_request));
+}
+
+TEST_F(MessagePumpTest, ReceiveRawInsteadOfMessage) {
+  TestRequest request;
+  EXPECT_OK(message_pump_.SendMessage(PacketType::kTest, request));
+
+  Buffer received_raw_data;
+  EXPECT_NOT_OK(message_pump_.ReceiveRawData(&received_raw_data));
+}
+
+TEST_F(MessagePumpTest, ReceiveMessageWrongType) {
+  TestRequest request;
+  EXPECT_OK(message_pump_.SendMessage(PacketType::kTest, request));
+
+  ShutdownRequest received_request;
+  EXPECT_NOT_OK(
+      message_pump_.ReceiveMessage(PacketType::kShutdown, &received_request));
+}
+
+TEST_F(MessagePumpTest, MessageMaxSizeExceeded) {
+  TestRequest request;
+  size_t max_size = message_pump_.GetMaxPacketSizeForTesting();
+  request.set_message(std::string(max_size + 1, 'x'));
+  EXPECT_NOT_OK(message_pump_.SendMessage(PacketType::kTest, request));
+}
+
+TEST_F(MessagePumpTest, FlushOutgoingQueue) {
+  TestRequest request;
+  request.set_message(std::string(1024 * 4, 'x'));
+  constexpr size_t kNumMessages = 1000;
+
+  // Note: Must stay below max queue size or else SendMessage starts blocking.
+  ASSERT_LT((request.message().size() + 4) * kNumMessages,
+            message_pump_.GetMaxInOutBufferSizeForTesting());
+
+  // Queue up a bunch of large messages.
+  fake_socket_.SuspendSending(true);
+  for (size_t n = 0; n < kNumMessages; ++n) {
+    EXPECT_OK(message_pump_.SendMessage(PacketType::kTest, request));
+  }
+  EXPECT_GT(message_pump_.GetNumOutgoingPackagesForTesting(), 0);
+
+  // Flush the queue.
+  fake_socket_.SuspendSending(false);
+  message_pump_.FlushOutgoingQueue();
+
+  // Check if the queue is empty.
+  EXPECT_EQ(message_pump_.GetNumOutgoingPackagesForTesting(), 0);
+}
+
+class FakeCompressedInputReader : public MessagePump::InputReader {
+ public:
+  explicit FakeCompressedInputReader(Socket* socket) : socket_(socket) {}
+
+  // Doesn't actually do compression, just replaces the word "compressed" by
+  // "COMPRESSED" as a sign that this handler was executed. In the real rsync
+  // algorithm, this is used to decompress data.
+  absl::Status Read(void* out_buffer, size_t out_size, size_t* bytes_read,
+                    bool* eof) override {
+    absl::Status status = socket_->Receive(
+        out_buffer, out_size, /*allow_partial_read=*/false, bytes_read);
+    if (!status.ok()) {
+      return WrapStatus(status, "socket_->Receive() failed");
+    }
+    assert(*bytes_read == out_size);
+    char* char_buffer = static_cast<char*>(out_buffer);
+    char* pos = strstr(char_buffer, "compressed");
+    if (pos) {
+      memcpy(pos, "COMPRESSED", strlen("COMPRESSED"));
+    }
+    *eof = strstr(char_buffer, "set_eof") != nullptr;
+    return absl::OkStatus();
+  };
+
+ private:
+  Socket* socket_;
+};
+
+TEST_F(MessagePumpTest, RedirectInput) {
+  fake_compressed_input_reader_ =
+      std::make_unique<FakeCompressedInputReader>(&fake_socket_);
+
+  TestRequest test_request;
+  ToggleCompressionRequest compression_request;
+
+  test_request.set_message("uncompressed");
+  EXPECT_OK(message_pump_.SendMessage(PacketType::kTest, test_request));
+
+  // Once this message is received, |fake_compressed_input_reader_| is set by
+  // ThreadPackageReceived().
+  EXPECT_OK(message_pump_.SendMessage(PacketType::kToggleCompression,
+                                      compression_request));
+
+  // Send a "compressed" message (should be converted to upper case).
+  test_request.set_message("compressed");
+  EXPECT_OK(message_pump_.SendMessage(PacketType::kTest, test_request));
+
+  // Trigger reset of the input reader.
+  test_request.set_message("set_eof");
+  EXPECT_OK(message_pump_.SendMessage(PacketType::kTest, test_request));
+
+  // The next message should be "uncompressed" (lower case) again.
+  test_request.set_message("uncompressed");
+  EXPECT_OK(message_pump_.SendMessage(PacketType::kTest, test_request));
+
+  EXPECT_OK(message_pump_.ReceiveMessage(PacketType::kTest, &test_request));
+  EXPECT_EQ(test_request.message(), "uncompressed");
+
+  EXPECT_OK(message_pump_.ReceiveMessage(PacketType::kToggleCompression,
+                                         &compression_request));
+
+  EXPECT_OK(message_pump_.ReceiveMessage(PacketType::kTest, &test_request));
+  EXPECT_EQ(test_request.message(), "COMPRESSED");
+
+  EXPECT_OK(message_pump_.ReceiveMessage(PacketType::kTest, &test_request));
+  EXPECT_EQ(test_request.message(), "set_eof");
+
+  EXPECT_OK(message_pump_.ReceiveMessage(PacketType::kTest, &test_request));
+  EXPECT_EQ(test_request.message(), "uncompressed");
+}
+
+TEST_F(MessagePumpTest, RedirectOutput) {
+  // Doesn't actually do compression, just replaces the word "compressed" by
+  // "COMPRESSED" as a sign that this handler was executed. In the real rsync
+  // algorithm, this handler would pipe the data through zstd to compress it.
+  auto fake_compressed_output_handler = [this](const void* data, size_t size) {
+    std::string char_buffer(static_cast<const char*>(data), size);
+    std::string::size_type pos = char_buffer.find("compressed");
+    if (pos != std::string::npos) {
+      char_buffer.replace(pos, strlen("COMPRESSED"), "COMPRESSED");
+    }
+    return fake_socket_.Send(char_buffer.data(), size);
+  };
+
+  TestRequest test_request;
+  ToggleCompressionRequest compression_request;
+
+  test_request.set_message("uncompressed");
+  EXPECT_OK(message_pump_.SendMessage(PacketType::kTest, test_request));
+
+  // Set output handler.
+  message_pump_.RedirectOutput(fake_compressed_output_handler);
+
+  // Send a "compressed" message (should be converted to upper case).
+  test_request.set_message("compressed");
+  EXPECT_OK(message_pump_.SendMessage(PacketType::kTest, test_request));
+
+  // Clear output handler again.
+  message_pump_.RedirectOutput(MessagePump::OutputHandler());
+
+  // The next message should be "uncompressed" (lower case) again.
+  test_request.set_message("uncompressed");
+  EXPECT_OK(message_pump_.SendMessage(PacketType::kTest, test_request));
+
+  EXPECT_OK(message_pump_.ReceiveMessage(PacketType::kTest, &test_request));
+  EXPECT_EQ(test_request.message(), "uncompressed");
+
+  EXPECT_OK(message_pump_.ReceiveMessage(PacketType::kTest, &test_request));
+  EXPECT_EQ(test_request.message(), "COMPRESSED");
+
+  EXPECT_OK(message_pump_.ReceiveMessage(PacketType::kTest, &test_request));
+  EXPECT_EQ(test_request.message(), "uncompressed");
+}
+
+}  // namespace
+}  // namespace cdc_ft

cdc_rsync/base/server_exit_code.h

@@ -0,0 +1,63 @@
+/*
+ * Copyright 2022 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef CDC_RSYNC_BASE_SERVER_EXIT_CODE_H_
+#define CDC_RSYNC_BASE_SERVER_EXIT_CODE_H_
+
+namespace cdc_ft {
+
+// Since the client cannot distinguish between stderr and stdout (ssh.exe sends
+// both to stdout), the server marks the beginning and ending of error messages
+// with this marker char. The client interprets everything in between as an
+// error message.
+constexpr char kServerErrorMarker = 0x1e;
+
+enum ServerExitCode {
+  // Pick a range of exit codes that does not overlap with unrelated exit codes
+  // like bash exit codes.
+  // - 126: error from bash when binary can't be started (permission denied).
+  // - 127: error from bash when binary isn't found
+  // - 255: ssh.exe error code.
+  // Note that codes must be <= 255.
+
+  // KEEP UPDATED!
+  kServerExitCodeMin = 50,
+
+  // Generic error on startup, before out-of-date check, e.g. bad args.
+  kServerExitCodeGenericStartup = 50,
+
+  // A gamelet component is outdated and needs to be re-uploaded.
+  kServerExitCodeOutOfDate = 51,
+
+  //
+  // All other exit codes must be strictly bigger than kServerErrorOutOfDate.
+  // They are guaranteed to be past the out-of-date check.
+  //
+
+  // Unspecified error.
+  kServerExitCodeGeneric = 52,
+
+  // Binding to the forward port failed, probably because there's another
+  // instance of cdc_rsync running.
+  kServerExitCodeAddressInUse = 53,
+
+  // KEEP UPDATED!
+  kServerExitCodeMax = 53,
+};
+
+}  // namespace cdc_ft
+
+#endif  // CDC_RSYNC_BASE_SERVER_EXIT_CODE_H_

cdc_rsync/base/socket.h

@@ -0,0 +1,45 @@
+/*
+ * Copyright 2022 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef CDC_RSYNC_BASE_SOCKET_H_
+#define CDC_RSYNC_BASE_SOCKET_H_
+
+#include "absl/status/status.h"
+
+namespace cdc_ft {
+
+class Socket {
+ public:
+  Socket() = default;
+  virtual ~Socket() = default;
+
+  // Send data to the socket.
+  virtual absl::Status Send(const void* buffer, size_t size) = 0;
+
+  // Receives data from the socket. Blocks until data is available or the
+  // sending end of the socket gets shut down by the sender.
+  // If |allow_partial_read| is false, blocks until |size| bytes are available.
+  // If |allow_partial_read| is true, may return with success if less than
+  // |size| (but more than 0) bytes were received.
+  // The number of bytes written to |buffer| is returned in |bytes_received|.
+  virtual absl::Status Receive(void* buffer, size_t size,
+                               bool allow_partial_read,
+                               size_t* bytes_received) = 0;
+};
+
+}  // namespace cdc_ft
+
+#endif  // CDC_RSYNC_BASE_SOCKET_H_

cdc_rsync/base/testdata/cdc_interface/new_file.txt

@@ -0,0 +1 @@
+Data for rsync testing. This is the new, modified file on the workstation.

cdc_rsync/base/testdata/cdc_interface/old_file.txt

@@ -0,0 +1 @@
+Data for rsync testing. This is the old version on the gamelet.

cdc_rsync/cdc_rsync.cc

@@ -0,0 +1,125 @@
+// Copyright 2022 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "cdc_rsync/cdc_rsync.h"
+
+#include <vector>
+
+#include "cdc_rsync/cdc_rsync_client.h"
+#include "cdc_rsync/error_messages.h"
+#include "common/log.h"
+#include "common/path_filter.h"
+#include "common/status.h"
+
+namespace cdc_ft {
+namespace {
+
+ReturnCode TagToMessage(Tag tag, const Options* options, std::string* msg) {
+  msg->clear();
+  switch (tag) {
+    case Tag::kSocketEof:
+      *msg = kMsgConnectionLost;
+      return ReturnCode::kConnectionLost;
+
+    case Tag::kAddressInUse:
+      *msg = kMsgAddressInUse;
+      return ReturnCode::kAddressInUse;
+
+    case Tag::kDeployServer:
+      *msg = kMsgDeployFailed;
+      return ReturnCode::kDeployFailed;
+
+    case Tag::kInstancePickerNotAvailableInQuietMode:
+      *msg = kMsgInstancePickerNotAvailableInQuietMode;
+      return ReturnCode::kInstancePickerNotAvailableInQuietMode;
+
+    case Tag::kConnectionTimeout:
+      *msg =
+          absl::StrFormat(kMsgFmtConnectionTimeout, options->ip, options->port);
+      return ReturnCode::kConnectionTimeout;
+
+    case Tag::kCount:
+      return ReturnCode::kGenericError;
+  }
+
+  // Should not happen (TM). Will fall back to status message in this case.
+  return ReturnCode::kGenericError;
+}
+
+PathFilter::Rule::Type ToInternalType(FilterRule::Type type) {
+  switch (type) {
+    case FilterRule::Type::kInclude:
+      return PathFilter::Rule::Type::kInclude;
+    case FilterRule::Type::kExclude:
+      return PathFilter::Rule::Type::kExclude;
+  }
+  assert(false);
+  return PathFilter::Rule::Type::kInclude;
+}
+
+}  // namespace
+
+ReturnCode Sync(const Options* options, const FilterRule* filter_rules,
+                size_t num_filter_rules, const char* sources_dir,
+                const char* const* sources, size_t num_sources,
+                const char* destination, const char** error_message) {
+  LogLevel log_level = Log::VerbosityToLogLevel(options->verbosity);
+  Log::Initialize(std::make_unique<ConsoleLog>(log_level));
+
+  PathFilter path_filter;
+  for (size_t n = 0; n < num_filter_rules; ++n) {
+    path_filter.AddRule(ToInternalType(filter_rules[n].type),
+                        filter_rules[n].pattern);
+  }
+
+  std::vector<std::string> sources_vec;
+  for (size_t n = 0; n < num_sources; ++n) {
+    sources_vec.push_back(sources[n]);
+  }
+
+  // Run rsync.
+  GgpRsyncClient client(*options, std::move(path_filter), sources_dir,
+                        std::move(sources_vec), destination);
+  absl::Status status = client.Run();
+
+  if (status.ok()) {
+    *error_message = nullptr;
+    return ReturnCode::kOk;
+  }
+
+  std::string msg;
+  ReturnCode code = ReturnCode::kGenericError;
+  absl::optional<Tag> tag = GetTag(status);
+  if (tag.has_value()) {
+    code = TagToMessage(tag.value(), options, &msg);
+  }
+
+  // Fall back to status message.
+  if (msg.empty()) {
+    msg = std::string(status.message());
+  } else if (options->verbosity >= 2) {
+    // In verbose mode, log the status as well, so nothing gets lost.
+    LOG_ERROR("%s", status.ToString().c_str());
+  }
+
+  // Store error message in static buffer (don't use std::string through DLL
+  // boundary!).
+  static char buf[1024] = {0};
+  strncpy_s(buf, msg.c_str(), _TRUNCATE);
+  *error_message = buf;
+
+  return code;
+}
+
+}  // namespace cdc_ft

cdc_rsync/cdc_rsync.h

@@ -0,0 +1,107 @@
+/*
+ * Copyright 2022 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef CDC_RSYNC_CDC_RSYNC_H_
+#define CDC_RSYNC_CDC_RSYNC_H_
+
+#ifdef COMPILING_DLL
+#define CDC_RSYNC_API __declspec(dllexport)
+#else
+#define CDC_RSYNC_API __declspec(dllimport)
+#endif
+
+namespace cdc_ft {
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+struct Options {
+  const char* ip = nullptr;
+  int port = 0;
+  bool delete_ = false;
+  bool recursive = false;
+  int verbosity = 0;
+  bool quiet = false;
+  bool whole_file = false;
+  bool relative = false;
+  bool compress = false;
+  bool checksum = false;
+  bool dry_run = false;
+  bool existing = false;
+  bool json = false;
+  const char* copy_dest = nullptr;
+  int compress_level = 6;
+  int connection_timeout_sec = 10;
+
+  // Compression level 0 is invalid.
+  static constexpr int kMinCompressLevel = -5;
+  static constexpr int kMaxCompressLevel = 22;
+};
+
+// Rule for including/excluding files.
+struct FilterRule {
+  enum class Type {
+    kInclude,
+    kExclude,
+  };
+
+  Type type;
+  const char* pattern;
+
+  FilterRule(Type type, const char* pattern) : type(type), pattern(pattern) {}
+};
+
+enum class ReturnCode {
+  // No error. Will match the tool's exit code, so OK must be 0.
+  kOk = 0,
+
+  // Generic error.
+  kGenericError = 1,
+
+  // Server connection timed out.
+  kConnectionTimeout = 2,
+
+  // Connection to the server was shut down unexpectedly.
+  kConnectionLost = 3,
+
+  // Binding to the forward port failed, probably because there's another
+  // instance of cdc_rsync running.
+  kAddressInUse = 4,
+
+  // Server deployment failed. This should be rare, it means that the server
+  // components were successfully copied, but the up-to-date check still fails.
+  kDeployFailed = 5,
+
+  // Gamelet selection asks for user input, but we are in quiet mode.
+  kInstancePickerNotAvailableInQuietMode = 6,
+};
+
+// Calling Sync() a second time overwrites the data in |error_message|.
+CDC_RSYNC_API ReturnCode Sync(const Options* options,
+                              const FilterRule* filter_rules,
+                              size_t filter_num_rules, const char* sources_dir,
+                              const char* const* sources, size_t num_sources,
+                              const char* destination,
+                              const char** error_message);
+
+#ifdef __cplusplus
+}  // extern "C"
+#endif
+
+}  // namespace cdc_ft
+
+#endif  // CDC_RSYNC_CDC_RSYNC_H_

cdc_rsync/cdc_rsync_client.cc

@@ -0,0 +1,789 @@
+// Copyright 2022 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "cdc_rsync/cdc_rsync_client.h"
+
+#include "absl/strings/str_format.h"
+#include "absl/strings/str_split.h"
+#include "cdc_rsync/base/cdc_interface.h"
+#include "cdc_rsync/base/message_pump.h"
+#include "cdc_rsync/base/server_exit_code.h"
+#include "cdc_rsync/client_file_info.h"
+#include "cdc_rsync/client_socket.h"
+#include "cdc_rsync/file_finder_and_sender.h"
+#include "cdc_rsync/parallel_file_opener.h"
+#include "cdc_rsync/progress_tracker.h"
+#include "cdc_rsync/protos/messages.pb.h"
+#include "cdc_rsync/zstd_stream.h"
+#include "common/gamelet_component.h"
+#include "common/log.h"
+#include "common/path.h"
+#include "common/process.h"
+#include "common/status.h"
+#include "common/status_macros.h"
+#include "common/stopwatch.h"
+#include "common/util.h"
+
+namespace cdc_ft {
+namespace {
+
+// Bash exit code if binary could not be run, e.g. permission denied.
+constexpr int kExitCodeCouldNotExecute = 126;
+
+// Bash exit code if binary was not found.
+constexpr int kExitCodeNotFound = 127;
+
+constexpr int kForwardPortFirst = 44450;
+constexpr int kForwardPortLast = 44459;
+constexpr char kGgpServerFilename[] = "cdc_rsync_server";
+constexpr char kRemoteToolsBinDir[] = "/opt/developer/tools/bin/";
+
+SetOptionsRequest::FilterRule::Type ToProtoType(PathFilter::Rule::Type type) {
+  switch (type) {
+    case PathFilter::Rule::Type::kInclude:
+      return SetOptionsRequest::FilterRule::TYPE_INCLUDE;
+    case PathFilter::Rule::Type::kExclude:
+      return SetOptionsRequest::FilterRule::TYPE_EXCLUDE;
+  }
+  assert(false);
+  return SetOptionsRequest::FilterRule::TYPE_INCLUDE;
+}
+
+// Translates a server process exit code and stderr into a status.
+absl::Status GetServerExitStatus(int exit_code, const std::string& error_msg) {
+  auto se_code = static_cast<ServerExitCode>(exit_code);
+  switch (se_code) {
+    case kServerExitCodeGenericStartup:
+      if (!error_msg.empty()) {
+        return MakeStatus("Server returned error during startup: %s",
+                          error_msg);
+      }
+      return MakeStatus(
+          "Server exited with an unspecified error during startup");
+
+    case kServerExitCodeOutOfDate:
+      return MakeStatus(
+          "Server exited since instance components are out of date");
+
+    case kServerExitCodeGeneric:
+      if (!error_msg.empty()) {
+        return MakeStatus("Server returned error: %s", error_msg);
+      }
+      return MakeStatus("Server exited with an unspecified error");
+
+    case kServerExitCodeAddressInUse:
+      return SetTag(MakeStatus("Server failed to connect"), Tag::kAddressInUse);
+  }
+
+  // Could potentially happen if the server exits due to another reason,
+  // e.g. some ssh.exe error (remember that the server process is actually
+  // an ssh process).
+  return MakeStatus("Server exited with code %i", exit_code);
+}
+
+}  // namespace
+
+GgpRsyncClient::GgpRsyncClient(const Options& options, PathFilter path_filter,
+                               std::string sources_dir,
+                               std::vector<std::string> sources,
+                               std::string destination)
+    : options_(options),
+      path_filter_(std::move(path_filter)),
+      sources_dir_(std::move(sources_dir)),
+      sources_(std::move(sources)),
+      destination_(std::move(destination)),
+      remote_util_(options.verbosity, options.quiet, &process_factory_,
+                   /*forward_output_to_log=*/false),
+      port_manager_("cdc_rsync_ports_f77bcdfe-368c-4c45-9f01-230c5e7e2132",
+                    kForwardPortFirst, kForwardPortLast, &process_factory_,
+                    &remote_util_),
+      printer_(options.quiet, Util::IsTTY() && !options.json),
+      progress_(&printer_, options.verbosity, options.json) {}
+
+GgpRsyncClient::~GgpRsyncClient() {
+  message_pump_.StopMessagePump();
+  socket_.Disconnect();
+}
+
+absl::Status GgpRsyncClient::Run() {
+  absl::Status status = remote_util_.GetInitStatus();
+  if (!status.ok()) {
+    return WrapStatus(status, "Failed to initialize critical components");
+  }
+
+  // Initialize |remote_util_|.
+  remote_util_.SetIpAndPort(options_.ip, options_.port);
+
+  // Start the server process.
+  status = StartServer();
+  if (HasTag(status, Tag::kDeployServer)) {
+    // Gamelet components are not deployed or out-dated. Deploy and retry.
+    status = DeployServer();
+    if (!status.ok()) {
+      return WrapStatus(status, "Failed to deploy server");
+    }
+
+    status = StartServer();
+  }
+  if (!status.ok()) {
+    return WrapStatus(status, "Failed to start server");
+  }
+
+  // Tag::kSocketEof most likely means that the server had an error exited. In
+  // that case, try to shut it down properly to get more info from the error
+  // message.
+  status = Sync();
+  if (!status.ok() && !HasTag(status, Tag::kSocketEof)) {
+    return WrapStatus(status, "Failed to sync files");
+  }
+
+  absl::Status stop_status = StopServer();
+  if (!stop_status.ok()) {
+    return WrapStatus(stop_status, "Failed to stop server");
+  }
+
+  // If the server doesn't send any error information, return the sync status.
+  if (server_error_.empty() && HasTag(status, Tag::kSocketEof)) {
+    return status;
+  }
+
+  // Check exit code and stderr.
+  if (server_exit_code_ != 0) {
+    status = GetServerExitStatus(server_exit_code_, server_error_);
+  }
+
+  return status;
+}
+
+absl::Status GgpRsyncClient::StartServer() {
+  assert(!server_process_);
+
+  // Components are expected to reside in the same dir as the executable.
+  std::string component_dir;
+  absl::Status status = path::GetExeDir(&component_dir);
+  if (!status.ok()) {
+    return WrapStatus(status, "Failed to get the executable directory");
+  }
+
+  std::vector<GameletComponent> components;
+  status = GameletComponent::Get(
+      {path::Join(component_dir, kGgpServerFilename)}, &components);
+  if (!status.ok()) {
+    return MakeStatus(
+        "Required instance component not found. Make sure the file "
+        "cdc_rsync_server resides in the same folder as cdc_rsync.exe.");
+  }
+  std::string component_args = GameletComponent::ToCommandLineArgs(components);
+
+  // Find available local and remote ports for port forwarding.
+  absl::StatusOr<int> port_res =
+      port_manager_.ReservePort(options_.connection_timeout_sec);
+  constexpr char kErrorMsg[] = "Failed to find available port";
+  if (absl::IsDeadlineExceeded(port_res.status())) {
+    // Server didn't respond in time.
+    return SetTag(WrapStatus(port_res.status(), kErrorMsg),
+                  Tag::kConnectionTimeout);
+  }
+  if (absl::IsResourceExhausted(port_res.status()))
+    return SetTag(WrapStatus(port_res.status(), kErrorMsg), Tag::kAddressInUse);
+  if (!port_res.ok())
+    return WrapStatus(port_res.status(), "Failed to find available port");
+  int port = *port_res;
+
+  std::string remote_server_path =
+      std::string(kRemoteToolsBinDir) + kGgpServerFilename;
+  // Test existence manually to prevent misleading bash output message
+  // "bash: .../cdc_rsync_server: No such file or directory".
+  std::string remote_command = absl::StrFormat(
+      "if [ ! -f %s ]; then exit %i; fi; %s %i %s", remote_server_path,
+      kExitCodeNotFound, remote_server_path, port, component_args);
+  ProcessStartInfo start_info =
+      remote_util_.BuildProcessStartInfoForSshPortForwardAndCommand(
+          port, port, false, remote_command);
+  start_info.name = "cdc_rsync_server";
+
+  // Capture stdout, but forward to stdout for debugging purposes.
+  start_info.stdout_handler = [this](const char* data, size_t /*data_size*/) {
+    return HandleServerOutput(data);
+  };
+
+  std::unique_ptr<Process> process = process_factory_.Create(start_info);
+  status = process->Start();
+  if (!status.ok()) {
+    return WrapStatus(status, "Failed to start cdc_rsync_server process");
+  }
+
+  // Wait until the server process is listening.
+  auto detect_listening = [is_listening = &is_server_listening_]() -> bool {
+    return *is_listening;
+  };
+  status = process->RunUntil(detect_listening);
+  if (!status.ok()) {
+    // Some internal process error. Note that this does NOT mean that
+    // cdc_rsync_server does not exist. In that case, the ssh process exits with
+    // code 127.
+    return status;
+  }
+
+  if (process->HasExited()) {
+    // Don't re-deploy for code > kServerExitCodeOutOfDate, which means that the
+    // out-of-date check already passed on the server.
+    server_exit_code_ = process->ExitCode();
+    if (server_exit_code_ > kServerExitCodeOutOfDate &&
+        server_exit_code_ <= kServerExitCodeMax) {
+      return GetServerExitStatus(server_exit_code_, server_error_);
+    }
+
+    // Server exited before it started listening, most likely because of
+    // outdated components (code kServerExitCodeOutOfDate) or because the server
+    // wasn't deployed at all yet (code kExitCodeNotFound). Instruct caller
+    // to re-deploy.
+    return SetTag(MakeStatus("Redeploy server"), Tag::kDeployServer);
+  }
+
+  assert(is_server_listening_);
+  status = socket_.Connect(port);
+  if (!status.ok()) {
+    return WrapStatus(status, "Failed to initialize connection");
+  }
+
+  server_process_ = std::move(process);
+  message_pump_.StartMessagePump();
+  return absl::OkStatus();
+}
+
+absl::Status GgpRsyncClient::StopServer() {
+  assert(server_process_);
+
+  // Close socket.
+  absl::Status status = socket_.ShutdownSendingEnd();
+  if (!status.ok()) {
+    return WrapStatus(status, "Failed to shut down socket sending end");
+  }
+
+  status = server_process_->RunUntilExit();
+  if (!status.ok()) {
+    return WrapStatus(status, "Failed to stop cdc_rsync_server process");
+  }
+
+  server_exit_code_ = server_process_->ExitCode();
+  server_process_.reset();
+  return absl::OkStatus();
+}
+
+absl::Status GgpRsyncClient::HandleServerOutput(const char* data) {
+  // Note: This is called from a background thread!
+
+  // Handle server error messages. Unfortunately, if the server prints to
+  // stderr, the ssh process does not write it to its stderr, but to stdout, so
+  // we have to jump through hoops to read the error. We use a marker char for
+  // the start of the error message:
+  //   This is stdout \x1e This is stderr \x1e This is stdout again
+  std::string stdout_data_storage;
+  const char* stdout_data = data;
+  if (is_server_error_ || strchr(data, kServerErrorMarker)) {
+    // Only run this expensive code if necessary.
+    std::vector<std::string> parts =
+        absl::StrSplit(data, absl::ByChar(kServerErrorMarker));
+    for (size_t n = 0; n < parts.size(); ++n) {
+      if (is_server_error_) {
+        server_error_.append(parts[n]);
+      } else {
+        stdout_data_storage.append(parts[n]);
+      }
+      if (n + 1 < parts.size()) {
+        is_server_error_ = !is_server_error_;
+      }
+    }
+    stdout_data = stdout_data_storage.c_str();
+  }
+
+  printer_.Print(stdout_data, false, Util::GetConsoleWidth());
+  if (!is_server_listening_) {
+    server_output_.append(stdout_data);
+    is_server_listening_ =
+        server_output_.find("Server is listening") != std::string::npos;
+  }
+
+  return absl::OkStatus();
+}
+
+absl::Status GgpRsyncClient::Sync() {
+  absl::Status status = SendOptions();
+  if (!status.ok()) {
+    return WrapStatus(status, "Failed to send options to server");
+  }
+
+  status = FindAndSendAllSourceFiles();
+  if (!status.ok()) {
+    return WrapStatus(status, "Failed to find and send all source files");
+  }
+
+  status = ReceiveFileStats();
+  if (!status.ok()) {
+    return WrapStatus(status, "Failed to receive file stats");
+  }
+
+  if (options_.delete_) {
+    status = ReceiveDeletedFiles();
+    if (!status.ok()) {
+      return WrapStatus(status, "Failed to receive paths of deleted files");
+    }
+  }
+
+  status = ReceiveFileIndices("missing", &missing_file_indices_);
+  if (!status.ok()) {
+    return WrapStatus(status, "Failed to receive missing file indices");
+  }
+  status = SendMissingFiles();
+  if (!status.ok()) {
+    return WrapStatus(status, "Failed to send missing files");
+  }
+
+  status = ReceiveFileIndices("changed", &changed_file_indices_);
+  if (!status.ok()) {
+    return WrapStatus(status, "Failed to receive changed file indices");
+  }
+
+  status = ReceiveSignaturesAndSendDelta();
+  if (!status.ok()) {
+    return WrapStatus(status, "Failed to receive signatures and send delta");
+  }
+
+  // Set sync point for shutdown (waits for the server to finish).
+  ShutdownRequest shutdown_request;
+  status = message_pump_.SendMessage(PacketType::kShutdown, shutdown_request);
+  if (!status.ok()) {
+    return WrapStatus(status, "Failed to send shutdown request");
+  }
+
+  ShutdownResponse response;
+  status = message_pump_.ReceiveMessage(PacketType::kShutdown, &response);
+  if (!status.ok()) {
+    return WrapStatus(status, "Failed to receive shutdown response");
+  }
+
+  return status;
+}
+
+absl::Status GgpRsyncClient::DeployServer() {
+  assert(!server_process_);
+
+  std::string exe_dir;
+  absl::Status status = path::GetExeDir(&exe_dir);
+  if (!status.ok()) {
+    return WrapStatus(status, "Failed to get exe directory");
+  }
+
+  std::string deploy_msg;
+  if (server_exit_code_ == kExitCodeNotFound) {
+    deploy_msg = "Server not deployed. Deploying...";
+  } else if (server_exit_code_ == kExitCodeCouldNotExecute) {
+    deploy_msg = "Server failed to start. Redeploying...";
+  } else if (server_exit_code_ == kServerExitCodeOutOfDate) {
+    deploy_msg = "Server outdated. Redeploying...";
+  } else {
+    deploy_msg = "Deploying server...";
+  }
+  printer_.Print(deploy_msg, true, Util::GetConsoleWidth());
+
+  // scp cdc_rsync_server to a temp location on the gamelet.
+  std::string remoteServerTmpPath =
+      absl::StrFormat("%s%s.%s", kRemoteToolsBinDir, kGgpServerFilename,
+                      Util::GenerateUniqueId());
+  std::string localServerPath = path::Join(exe_dir, kGgpServerFilename);
+  status = remote_util_.Scp({localServerPath}, remoteServerTmpPath,
+                            /*compress=*/true);
+  if (!status.ok()) {
+    return WrapStatus(status, "Failed to copy cdc_rsync_server to instance");
+  }
+
+  // Make cdc_rsync_server executable.
+  status = remote_util_.Chmod("a+x", remoteServerTmpPath);
+  if (!status.ok()) {
+    return WrapStatus(status,
+                      "Failed to set executable flag on cdc_rsync_server");
+  }
+
+  // Make old file writable. Mv might fail to overwrite it, e.g. if someone made
+  // it read-only.
+  std::string remoteServerPath =
+      std::string(kRemoteToolsBinDir) + kGgpServerFilename;
+  status = remote_util_.Chmod("u+w", remoteServerPath, /*quiet=*/true);
+  if (!status.ok()) {
+    LOG_DEBUG("chmod u+w %s failed (expected if file does not exist): %s",
+              remoteServerPath, status.ToString());
+  }
+
+  // Replace old file by new file.
+  status = remote_util_.Mv(remoteServerTmpPath, remoteServerPath);
+  if (!status.ok()) {
+    return WrapStatus(status, "Failed to replace '%s' by '%s'",
+                      remoteServerPath, remoteServerTmpPath);
+  }
+
+  return absl::OkStatus();
+}
+
+absl::Status GgpRsyncClient::SendOptions() {
+  LOG_INFO("Sending options");
+
+  SetOptionsRequest request;
+  request.set_destination(destination_);
+  request.set_delete_(options_.delete_);
+  request.set_recursive(options_.recursive);
+  request.set_verbosity(options_.verbosity);
+  request.set_whole_file(options_.whole_file);
+  request.set_compress(options_.compress);
+  request.set_relative(options_.relative);
+
+  for (const PathFilter::Rule& rule : path_filter_.GetRules()) {
+    SetOptionsRequest::FilterRule* filter_rule = request.add_filter_rules();
+    filter_rule->set_type(ToProtoType(rule.type));
+    filter_rule->set_pattern(rule.pattern);
+  }
+
+  request.set_checksum(options_.checksum);
+  request.set_dry_run(options_.dry_run);
+  request.set_existing(options_.existing);
+  if (options_.copy_dest) {
+    request.set_copy_dest(options_.copy_dest);
+  }
+
+  absl::Status status =
+      message_pump_.SendMessage(PacketType::kSetOptions, request);
+  if (!status.ok()) {
+    return WrapStatus(status, "SendDestination() failed");
+  }
+
+  return absl::OkStatus();
+}
+
+absl::Status GgpRsyncClient::FindAndSendAllSourceFiles() {
+  LOG_INFO("Finding and sending all sources files");
+
+  Stopwatch stopwatch;
+
+  FileFinderAndSender file_finder(&path_filter_, &message_pump_, &progress_,
+                                  sources_dir_, options_.recursive,
+                                  options_.relative);
+
+  progress_.StartFindFiles();
+  for (const std::string& source : sources_) {
+    absl::Status status = file_finder.FindAndSendFiles(source);
+    if (!status.ok()) {
+      return status;
+    }
+  }
+  progress_.Finish();
+
+  RETURN_IF_ERROR(file_finder.Flush(), "Failed to flush file finder");
+  file_finder.ReleaseFiles(&files_);
+
+  LOG_INFO("Found and sent %u source files in %0.3f seconds", files_.size(),
+           stopwatch.ElapsedSeconds());
+
+  return absl::OkStatus();
+}
+
+absl::Status GgpRsyncClient::ReceiveFileStats() {
+  LOG_INFO("Receiving file stats");
+
+  SendFileStatsResponse response;
+  absl::Status status =
+      message_pump_.ReceiveMessage(PacketType::kSendFileStats, &response);
+  if (!status.ok()) {
+    return WrapStatus(status, "Failed to receive SendFileStatsResponse");
+  }
+
+  progress_.ReportFileStats(
+      response.num_missing_files(), response.num_extraneous_files(),
+      response.num_matching_files(), response.num_changed_files(),
+      response.total_missing_bytes(), response.total_changed_client_bytes(),
+      response.total_changed_server_bytes(), response.num_missing_dirs(),
+      response.num_extraneous_dirs(), response.num_matching_dirs(),
+      options_.whole_file, options_.checksum, options_.delete_);
+  return absl::OkStatus();
+}
+
+absl::Status GgpRsyncClient::ReceiveDeletedFiles() {
+  LOG_INFO("Receiving path of deleted files");
+  std::string current_directory;
+
+  progress_.StartDeleteFiles();
+  for (;;) {
+    AddDeletedFilesResponse response;
+    absl::Status status =
+        message_pump_.ReceiveMessage(PacketType::kAddDeletedFiles, &response);
+    if (!status.ok()) {
+      return WrapStatus(status, "Failed to receive AddDeletedFilesResponse");
+    }
+
+    // An empty response indicates that all files have been sent.
+    if (response.files_size() == 0 && response.dirs_size() == 0) {
+      break;
+    }
+
+    // Print info. Don't use path::Join(), it would mess up slashes.
+    for (const std::string& file : response.files()) {
+      progress_.ReportFileDeleted(response.directory() + file);
+    }
+    for (const std::string& dir : response.dirs()) {
+      progress_.ReportDirDeleted(response.directory() + dir);
+    }
+  }
+  progress_.Finish();
+
+  return absl::OkStatus();
+}
+
+absl::Status GgpRsyncClient::ReceiveFileIndices(
+    const char* file_type, std::vector<uint32_t>* file_indices) {
+  LOG_INFO("Receiving indices of %s files", file_type);
+
+  for (;;) {
+    AddFileIndicesResponse response;
+    absl::Status status =
+        message_pump_.ReceiveMessage(PacketType::kAddFileIndices, &response);
+    if (!status.ok()) {
+      return WrapStatus(status, "Failed to receive AddFileIndicesResponse");
+    }
+
+    // An empty response indicates that all files have been sent.
+    if (response.client_indices_size() == 0) {
+      break;
+    }
+
+    // Record file indices.
+    file_indices->insert(file_indices->end(), response.client_indices().begin(),
+                         response.client_indices().end());
+  }
+
+  // Validate indices.
+  for (uint32_t index : *file_indices) {
+    if (index >= files_.size()) {
+      return MakeStatus("Received invalid index %u", index);
+    }
+  }
+
+  LOG_INFO("Received %u indices of %s files", file_indices->size(), file_type);
+
+  return absl::OkStatus();
+}
+
+absl::Status GgpRsyncClient::SendMissingFiles() {
+  if (missing_file_indices_.empty()) {
+    return absl::OkStatus();
+  }
+
+  LOG_INFO("Sending missing files");
+
+  if (options_.dry_run) {
+    for (uint32_t client_index : missing_file_indices_) {
+      const ClientFileInfo& file = files_[client_index];
+      progress_.StartCopy(file.path.substr(file.base_dir_len), file.size);
+      progress_.Finish();
+    }
+    return absl::OkStatus();
+  }
+
+  // This part is (optionally) compressed.
+  if (options_.compress) {
+    absl::Status status = StartCompressionStream();
+    if (!status.ok()) {
+      return WrapStatus(status, "Failed to start compression process");
+    }
+  }
+
+  ParallelFileOpener file_opener(&files_, missing_file_indices_);
+
+  constexpr size_t kBufferSize = 16000;
+  for (uint32_t server_index = 0; server_index < missing_file_indices_.size();
+       ++server_index) {
+    uint32_t client_index = missing_file_indices_[server_index];
+    const ClientFileInfo& file = files_[client_index];
+
+    LOG_INFO("%s", file.path);
+    progress_.StartCopy(file.path.substr(file.base_dir_len), file.size);
+    SendMissingFileDataRequest request;
+    request.set_server_index(server_index);
+    absl::Status status =
+        message_pump_.SendMessage(PacketType::kSendMissingFileData, request);
+    if (!status.ok()) {
+      return WrapStatus(status, "Failed to send SendMissingFileDataRequest");
+    }
+    ProgressTracker* progress = &progress_;
+    auto handler = [message_pump = &message_pump_, progress](const void* data,
+                                                             size_t size) {
+      progress->ReportCopyProgress(size);
+      return message_pump->SendRawData(data, size);
+    };
+
+    FILE* fp = file_opener.GetNextOpenFile();
+    if (!fp) {
+      return MakeStatus("Failed to open file '%s'", file.path);
+    }
+    status = path::StreamReadFileContents(fp, kBufferSize, handler);
+    fclose(fp);
+    if (!status.ok()) {
+      return WrapStatus(status, "Failed to read file %s", file.path);
+    }
+
+    progress_.Finish();
+  }
+
+  if (options_.compress) {
+    absl::Status status = StopCompressionStream();
+    if (!status.ok()) {
+      return WrapStatus(status, "Failed to stop compression process");
+    }
+  }
+
+  return absl::OkStatus();
+}
+
+absl::Status GgpRsyncClient::ReceiveSignaturesAndSendDelta() {
+  if (changed_file_indices_.empty()) {
+    return absl::OkStatus();
+  }
+
+  if (options_.dry_run) {
+    for (uint32_t client_index : changed_file_indices_) {
+      const ClientFileInfo& file = files_[client_index];
+      progress_.StartSync(file.path.substr(file.base_dir_len), file.size,
+                          file.size);
+      progress_.ReportSyncProgress(file.size, file.size);
+      progress_.Finish();
+    }
+    return absl::OkStatus();
+  }
+
+  LOG_INFO("Receiving signatures and sending deltas of changed files");
+
+  // This part is (optionally) compressed.
+  if (options_.compress) {
+    absl::Status status = StartCompressionStream();
+    if (!status.ok()) {
+      return WrapStatus(status, "Failed to start compression process");
+    }
+  }
+
+  CdcInterface cdc(&message_pump_);
+
+  // Open files in parallel. Speeds up many small file case.
+  ParallelFileOpener file_opener(&files_, changed_file_indices_);
+
+  std::string signature_data;
+  for (uint32_t server_index = 0; server_index < changed_file_indices_.size();
+       ++server_index) {
+    uint32_t client_index = changed_file_indices_[server_index];
+    const ClientFileInfo& file = files_[client_index];
+
+    SendSignatureResponse response;
+    absl::Status status =
+        message_pump_.ReceiveMessage(PacketType::kAddSignatures, &response);
+    if (!status.ok()) {
+      return WrapStatus(status, "Failed to receive SendSignatureResponse");
+    }
+
+    // Validate index.
+    if (response.client_index() != client_index) {
+      return MakeStatus("Received invalid index %u. Expected %u.",
+                        response.client_index(), client_index);
+    }
+
+    LOG_INFO("%s", file.path);
+    progress_.StartSync(file.path.substr(file.base_dir_len), file.size,
+                        response.server_file_size());
+
+    FILE* fp = file_opener.GetNextOpenFile();
+    if (!fp) {
+      return MakeStatus("Failed to open file '%s'", file.path);
+    }
+
+    status = cdc.ReceiveSignatureAndCreateAndSendDiff(fp, &progress_);
+    fclose(fp);
+    if (!status.ok()) {
+      return WrapStatus(status, "Failed to sync file %s", file.path);
+    }
+
+    progress_.Finish();
+  }
+
+  if (options_.compress) {
+    absl::Status status = StopCompressionStream();
+    if (!status.ok()) {
+      return WrapStatus(status, "Failed to stop compression process");
+    }
+  }
+
+  return absl::OkStatus();
+}
+
+absl::Status GgpRsyncClient::StartCompressionStream() {
+  assert(!compression_stream_);
+
+  // Notify server that data is compressed from now on.
+  ToggleCompressionRequest request;
+  absl::Status status =
+      message_pump_.SendMessage(PacketType::kToggleCompression, request);
+  if (!status.ok()) {
+    return WrapStatus(status, "Failed to send ToggleCompressionRequest");
+  }
+
+  // Make sure the sender thread is idle.
+  message_pump_.FlushOutgoingQueue();
+
+  // Set up compression stream.
+  uint32_t num_threads = std::thread::hardware_concurrency();
+  compression_stream_ = std::make_unique<ZstdStream>(
+      &socket_, options_.compress_level, num_threads);
+
+  // Redirect the |message_pump_| output to the compression stream.
+  message_pump_.RedirectOutput([this](const void* data, size_t size) {
+    LOG_VERBOSE("Compressing packet of size %u", size);
+    return compression_stream_->Write(data, size);
+  });
+
+  // The pipes are now set up like this:
+  // |message_pump_| -> |compression_stream_| -> |socket_|.
+
+  return absl::OkStatus();
+}
+
+absl::Status GgpRsyncClient::StopCompressionStream() {
+  assert(compression_stream_);
+
+  // Finish writing to |compression_process_|'s stdin and change back to
+  // writing to the actual network socket.
+  message_pump_.FlushOutgoingQueue();
+  message_pump_.RedirectOutput(nullptr);
+
+  // Flush compression stream and reset.
+  RETURN_IF_ERROR(compression_stream_->Flush(),
+                  "Failed to flush compression stream");
+  compression_stream_.reset();
+
+  // Wait for the server ack. This must be done before sending more data.
+  ToggleCompressionResponse response;
+  absl::Status status =
+      message_pump_.ReceiveMessage(PacketType::kToggleCompression, &response);
+  if (!status.ok()) {
+    return WrapStatus(status, "Failed to receive ToggleCompressionResponse");
+  }
+
+  return absl::OkStatus();
+}
+
+}  // namespace cdc_ft

cdc_rsync/cdc_rsync_client.h

@@ -0,0 +1,132 @@
+/*
+ * Copyright 2022 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef CDC_RSYNC_CDC_RSYNC_CLIENT_H_
+#define CDC_RSYNC_CDC_RSYNC_CLIENT_H_
+
+#include <string>
+#include <vector>
+
+#include "absl/status/status.h"
+#include "cdc_rsync/base/message_pump.h"
+#include "cdc_rsync/cdc_rsync.h"
+#include "cdc_rsync/client_socket.h"
+#include "cdc_rsync/progress_tracker.h"
+#include "common/path_filter.h"
+#include "common/port_manager.h"
+#include "common/remote_util.h"
+
+namespace cdc_ft {
+
+class Process;
+class ZstdStream;
+
+class GgpRsyncClient {
+ public:
+  GgpRsyncClient(const Options& options, PathFilter filter,
+                 std::string sources_dir, std::vector<std::string> sources,
+                 std::string destination);
+
+  ~GgpRsyncClient();
+
+  // Deploys the server if necessary, starts it and runs the rsync procedure.
+  absl::Status Run();
+
+ private:
+  // Starts the server process. If the method returns a status with tag
+  // |kTagDeployServer|, Run() calls DeployServer() and tries again.
+  absl::Status StartServer();
+
+  // Stops the server process.
+  absl::Status StopServer();
+
+  // Handler for stdout and stderr data emitted by the server.
+  absl::Status HandleServerOutput(const char* data);
+
+  // Runs the rsync procedure.
+  absl::Status Sync();
+
+  // Copies all gamelet components to the gamelet.
+  absl::Status DeployServer();
+
+  // Sends relevant options to the server.
+  absl::Status SendOptions();
+
+  // Finds all source files and sends the file infos to the server.
+  absl::Status FindAndSendAllSourceFiles();
+
+  // Receives the stats from the file diffs (e.g. number of missing, changed
+  // etc. files) from the server.
+  absl::Status ReceiveFileStats();
+
+  // Receives paths of deleted files and prints them out.
+  absl::Status ReceiveDeletedFiles();
+
+  // Receives file indices from the server. Used for missing and changed files.
+  absl::Status ReceiveFileIndices(const char* file_type,
+                                  std::vector<uint32_t>* file_indices);
+
+  // Copies missing files to the server.
+  absl::Status SendMissingFiles();
+
+  // Core rsync algorithm. Receives signatures of changed files from server,
+  // calculates the diffs and sends them to the server.
+  absl::Status ReceiveSignaturesAndSendDelta();
+
+  // Start the zstd compression stream. Used before file copy and diff.
+  absl::Status StartCompressionStream();
+
+  // Stops the zstd compression stream.
+  absl::Status StopCompressionStream();
+
+  Options options_;
+  PathFilter path_filter_;
+  const std::string sources_dir_;
+  std::vector<std::string> sources_;
+  const std::string destination_;
+
+  WinProcessFactory process_factory_;
+  RemoteUtil remote_util_;
+  PortManager port_manager_;
+  ClientSocket socket_;
+  MessagePump message_pump_{&socket_, MessagePump::PacketReceivedDelegate()};
+  ConsoleProgressPrinter printer_;
+  ProgressTracker progress_;
+  std::unique_ptr<ZstdStream> compression_stream_;
+
+  std::unique_ptr<Process> server_process_;
+  std::string server_output_;  // Written in a background thread. Do not access
+  std::string server_error_;   // while the server process is active.
+  int server_exit_code_ = 0;
+  std::atomic_bool is_server_listening_{false};
+  bool is_server_error_ = false;
+
+  // All source files found on the client.
+  std::vector<ClientFileInfo> files_;
+
+  // All source dirs found on the client.
+  std::vector<ClientDirInfo> dirs_;
+
+  // Indices (into files_) of files that are missing on the server.
+  std::vector<uint32_t> missing_file_indices_;
+
+  // Indices (into files_) of files that exist, but are different on the server.
+  std::vector<uint32_t> changed_file_indices_;
+};
+
+}  // namespace cdc_ft
+
+#endif  // CDC_RSYNC_CDC_RSYNC_CLIENT_H_

cdc_rsync/client_file_info.h

@@ -0,0 +1,43 @@
+/*
+ * Copyright 2022 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef CDC_RSYNC_CLIENT_FILE_INFO_H_
+#define CDC_RSYNC_CLIENT_FILE_INFO_H_
+
+#include <string>
+
+namespace cdc_ft {
+
+struct ClientFileInfo {
+  std::string path;
+  uint64_t size;
+  uint32_t base_dir_len;
+
+  ClientFileInfo(const std::string& path, uint64_t size, uint32_t base_dir_len)
+      : path(path), size(size), base_dir_len(base_dir_len) {}
+};
+
+struct ClientDirInfo {
+  std::string path;
+  uint32_t base_dir_len;
+
+  ClientDirInfo(const std::string& path, uint32_t base_dir_len)
+      : path(path), base_dir_len(base_dir_len) {}
+};
+
+}  // namespace cdc_ft
+
+#endif  // CDC_RSYNC_CLIENT_FILE_INFO_H_

cdc_rsync/client_socket.cc

@@ -0,0 +1,174 @@
+// Copyright 2022 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "cdc_rsync/client_socket.h"
+
+#include <winsock2.h>
+#include <ws2tcpip.h>
+
+#include <cassert>
+
+#include "common/log.h"
+#include "common/status.h"
+#include "common/util.h"
+
+namespace cdc_ft {
+namespace {
+
+// Creates a status with the given |message| and the last WSA error.
+// Assigns Tag::kSocketEof for WSAECONNRESET errors.
+absl::Status MakeSocketStatus(const char* message) {
+  const int err = WSAGetLastError();
+  absl::Status status = MakeStatus("%s: %s", message, Util::GetWin32Error(err));
+  if (err == WSAECONNRESET) {
+    status = SetTag(status, Tag::kSocketEof);
+  }
+  return status;
+}
+
+}  // namespace
+
+struct SocketInfo {
+  SOCKET socket;
+
+  SocketInfo() : socket(INVALID_SOCKET) {}
+};
+
+ClientSocket::ClientSocket() = default;
+
+ClientSocket::~ClientSocket() { Disconnect(); }
+
+absl::Status ClientSocket::Connect(int port) {
+  WSADATA wsaData;
+  int result = WSAStartup(MAKEWORD(2, 2), &wsaData);
+  if (result != 0) {
+    return MakeStatus("WSAStartup() failed: %i", result);
+  }
+
+  addrinfo hints;
+  ZeroMemory(&hints, sizeof(hints));
+  hints.ai_family = AF_INET;
+  hints.ai_socktype = SOCK_STREAM;
+  hints.ai_protocol = IPPROTO_TCP;
+
+  // Resolve the server address and port.
+  addrinfo* addr_infos = nullptr;
+  result = getaddrinfo("localhost", std::to_string(port).c_str(), &hints,
+                       &addr_infos);
+  if (result != 0) {
+    WSACleanup();
+    return MakeStatus("getaddrinfo() failed: %i", result);
+  }
+
+  socket_info_ = std::make_unique<SocketInfo>();
+  int count = 0;
+  for (addrinfo* curr = addr_infos; curr; curr = curr->ai_next, count++) {
+    socket_info_->socket =
+        socket(addr_infos->ai_family, addr_infos->ai_socktype,
+               addr_infos->ai_protocol);
+    if (socket_info_->socket == INVALID_SOCKET) {
+      LOG_DEBUG("socket() failed for addr_info %i: %s", count,
+                Util::GetWin32Error(WSAGetLastError()).c_str());
+      continue;
+    }
+
+    // Connect to server.
+    result = connect(socket_info_->socket, curr->ai_addr,
+                     static_cast<int>(curr->ai_addrlen));
+    if (result == SOCKET_ERROR) {
+      LOG_DEBUG("connect() failed for addr_info %i: %i", count, result);
+      closesocket(socket_info_->socket);
+      socket_info_->socket = INVALID_SOCKET;
+      continue;
+    }
+
+    // Success!
+    break;
+  }
+
+  freeaddrinfo(addr_infos);
+
+  if (socket_info_->socket == INVALID_SOCKET) {
+    socket_info_.reset();
+    WSACleanup();
+    return MakeStatus("Unable to connect to port %i", port);
+  }
+
+  LOG_INFO("Client socket connected to port %i", port);
+  return absl::OkStatus();
+}
+
+void ClientSocket::Disconnect() {
+  if (!socket_info_) {
+    return;
+  }
+
+  if (socket_info_->socket != INVALID_SOCKET) {
+    closesocket(socket_info_->socket);
+    socket_info_->socket = INVALID_SOCKET;
+  }
+
+  socket_info_.reset();
+  WSACleanup();
+}
+
+absl::Status ClientSocket::Send(const void* buffer, size_t size) {
+  int result = send(socket_info_->socket, static_cast<const char*>(buffer),
+                    static_cast<int>(size), /*flags */ 0);
+  if (result == SOCKET_ERROR) {
+    return MakeSocketStatus("send() failed");
+  }
+
+  return absl::OkStatus();
+}
+
+absl::Status ClientSocket::Receive(void* buffer, size_t size,
+                                   bool allow_partial_read,
+                                   size_t* bytes_received) {
+  *bytes_received = 0;
+  if (size == 0) {
+    return absl::OkStatus();
+  }
+
+  int flags = allow_partial_read ? 0 : MSG_WAITALL;
+  int bytes_read = recv(socket_info_->socket, static_cast<char*>(buffer),
+                        static_cast<int>(size), flags);
+  if (bytes_read == SOCKET_ERROR) {
+    return MakeSocketStatus("recv() failed");
+  }
+
+  if (bytes_read == 0) {
+    // EOF
+    return SetTag(MakeStatus("EOF detected"), Tag::kSocketEof);
+  }
+
+  if (bytes_read != size && !allow_partial_read) {
+    // Can this happen?
+    return MakeStatus("Partial read");
+  }
+
+  *bytes_received = bytes_read;
+  return absl::OkStatus();
+}
+
+absl::Status ClientSocket::ShutdownSendingEnd() {
+  int result = shutdown(socket_info_->socket, SD_SEND);
+  if (result == SOCKET_ERROR) {
+    return MakeSocketStatus("shutdown() failed");
+  }
+
+  return absl::OkStatus();
+}
+
+}  // namespace cdc_ft

cdc_rsync/client_socket.h

@@ -0,0 +1,53 @@
+/*
+ * Copyright 2022 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef CDC_RSYNC_CLIENT_SOCKET_H_
+#define CDC_RSYNC_CLIENT_SOCKET_H_
+
+#include <memory>
+
+#include "absl/status/status.h"
+#include "cdc_rsync/base/socket.h"
+
+namespace cdc_ft {
+
+class ClientSocket : public Socket {
+ public:
+  ClientSocket();
+  ~ClientSocket();
+
+  // Connects to localhost on |port|.
+  absl::Status Connect(int port);
+
+  // Disconnects again. No-op if not connected.
+  void Disconnect();
+
+  // Shuts down the sending end of the socket. This will interrupt any receive
+  // calls on the server and shut it down.
+  absl::Status ShutdownSendingEnd();
+
+  // Socket:
+  absl::Status Send(const void* buffer, size_t size) override;
+  absl::Status Receive(void* buffer, size_t size, bool allow_partial_read,
+                       size_t* bytes_received) override;
+
+ private:
+  std::unique_ptr<struct SocketInfo> socket_info_;
+};
+
+}  // namespace cdc_ft
+
+#endif  // CDC_RSYNC_CLIENT_SOCKET_H_

cdc_rsync/cpp.hint

@@ -0,0 +1,2 @@
+#define CDC_RSYNC_API __declspec(dllexport)
+#define CDC_RSYNC_API __declspec(dllimport)

cdc_rsync/dllmain.cc

@@ -0,0 +1,29 @@
+// Copyright 2022 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#define WIN32_LEAN_AND_MEAN
+#include <windows.h>
+
+BOOL APIENTRY DllMain(HMODULE /* hModule */, DWORD ul_reason_for_call,
+                      LPVOID /* lpReserved */
+) {
+  switch (ul_reason_for_call) {
+    case DLL_PROCESS_ATTACH:
+    case DLL_THREAD_ATTACH:
+    case DLL_THREAD_DETACH:
+    case DLL_PROCESS_DETACH:
+      break;
+  }
+  return TRUE;
+}

cdc_rsync/error_messages.h

@@ -0,0 +1,54 @@
+/*
+ * Copyright 2022 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef CDC_RSYNC_ERROR_MESSAGES_H_
+#define CDC_RSYNC_ERROR_MESSAGES_H_
+
+namespace cdc_ft {
+
+// Server connection timed out. SSH probably stale.
+constexpr char kMsgFmtConnectionTimeout[] =
+    "Server connection timed out. Please re-run 'ggp ssh init' and verify that "
+    "the IP '%s' and the port '%i' are correct.";
+
+// Server connection timed out and IP was not passed in. Probably network error.
+constexpr char kMsgConnectionTimeoutWithIp[] =
+    "Server connection timed out. Please check your network connection.";
+
+// Receiving pipe end was shut down unexpectedly.
+constexpr char kMsgConnectionLost[] =
+    "The connection to the instance was shut down unexpectedly.";
+
+// Binding to the port failed.
+constexpr char kMsgAddressInUse[] =
+    "Failed to establish a connection to the instance. All ports are already "
+    "in use. This can happen if another instance of this command is running. "
+    "Currently, only 10 simultaneous connections are supported.";
+
+// Deployment failed even though gamelet components were copied successfully.
+constexpr char kMsgDeployFailed[] =
+    "Failed to deploy the instance components for unknown reasons. "
+    "Please report this issue.";
+
+// Picking an instance is not allowed in quiet mode.
+constexpr char kMsgInstancePickerNotAvailableInQuietMode[] =
+    "Multiple gamelet instances are reserved, but the instance picker is not "
+    "available in quiet mode. Please specify --instance or remove -q resp. "
+    "--quiet.";
+
+}  // namespace cdc_ft
+
+#endif  // CDC_RSYNC_ERROR_MESSAGES_H_

cdc_rsync/file_finder_and_sender.cc

@@ -0,0 +1,248 @@
+// Copyright 2022 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "cdc_rsync/file_finder_and_sender.h"
+
+#include "absl/strings/match.h"
+#include "absl/strings/str_format.h"
+#include "cdc_rsync/base/message_pump.h"
+#include "common/log.h"
+#include "common/path.h"
+#include "common/path_filter.h"
+#include "common/status.h"
+
+namespace cdc_ft {
+namespace {
+
+bool EndsWithSpecialDir(const std::string& source) {
+  return source == "." || source == ".." || absl::EndsWith(source, "\\.") ||
+         absl::EndsWith(source, "\\..");
+}
+
+// Returns C:\ from C:\path\to\file or an empty string if there is no drive.
+std::string GetDrivePrefixWithBackslash(const std::string& source) {
+  std::string prefix = path::GetDrivePrefix(source);
+  if (source[prefix.size()] == '\\') {
+    prefix += "\\";
+  }
+  return prefix;
+}
+
+// Basically returns |sources_dir| + |source|, but removes drive letters from
+// |source| if present and |sources_dir| is not empty.
+std::string GetFullSource(const std::string& source,
+                          const std::string& sources_dir) {
+  if (sources_dir.empty()) {
+    return source;
+  }
+
+  // Combine |sources_dir_| and |source|, but remove the drive prefix, so
+  // that we don't get stuff like "source_dir\C:\path\to\file".
+  return path::Join(sources_dir,
+                    source.substr(GetDrivePrefixWithBackslash(source).size()));
+}
+
+std::string GetBaseDir(const std::string& source,
+                       const std::string& sources_dir, bool relative) {
+  if (!relative) {
+    // For non-relative mode, the base dir is the directory part, so that
+    // path\to\file is copied to remote_dir/file and files in path\to\ are
+    // copied to remote_dir.
+    if (path::EndsWithPathSeparator(source)) return source;
+    std::string dir = path::DirName(source);
+    if (!dir.empty()) path::EnsureEndsWithPathSeparator(&dir);
+    return dir;
+  }
+
+  // A "\.\" is a marker for where the relative path should start.
+  // The base dir is the part up to that marker, so that
+  // path\.\to\file is copied to remote_dir/to/file.
+  size_t pos = source.find("\\.\\");
+  if (pos != std::string::npos) {
+    return source.substr(0, pos + 3);
+  }
+
+  // If there is a sources dir, the base dir is the sources dir, so that
+  // sources_dir\path\to\file is copied to remote_dir/path/to/file.
+  if (!sources_dir.empty()) {
+    assert(source.find(sources_dir) == 0);
+    return sources_dir;
+  }
+
+  // If there is a drive prefix, the base dir is that part, so that
+  // C:\path\to\file is copied to remote_dir/path/to/file.
+  return GetDrivePrefixWithBackslash(source);
+}
+
+}  // namespace
+
+FileFinderAndSender::FileFinderAndSender(PathFilter* path_filter,
+                                         MessagePump* message_pump,
+                                         ReportFindFilesProgress* progress,
+                                         std::string sources_dir,
+                                         bool recursive, bool relative,
+                                         size_t request_byte_threshold)
+    : path_filter_(path_filter),
+      message_pump_(message_pump),
+      progress_(progress),
+      sources_dir_(std::move(sources_dir)),
+      recursive_(recursive),
+      relative_(relative),
+      request_size_threshold_(request_byte_threshold) {
+  // (internal): Support / instead of \ in the source folder.
+  path::FixPathSeparators(&sources_dir_);
+}
+
+FileFinderAndSender::~FileFinderAndSender() = default;
+
+absl::Status FileFinderAndSender::FindAndSendFiles(std::string source) {
+  // (internal): Support / instead of \ in sources.
+  path::FixPathSeparators(&source);
+  // Special case, "." and ".." should not specify the directory, but the files
+  // inside this directory!
+  if (EndsWithSpecialDir(source)) {
+    path::EnsureEndsWithPathSeparator(&source);
+  }
+
+  // Combine |source| and |sources_dir_| if present.
+  std::string full_source = GetFullSource(source, sources_dir_);
+
+  // Get the part of the path to remove before sending it to the server.
+  base_dir_ = GetBaseDir(full_source, sources_dir_, relative_);
+
+  size_t prev_size = files_.size() + dirs_.size();
+
+  auto handler = [this](std::string dir, std::string filename,
+                        int64_t modified_time, uint64_t size,
+                        bool is_directory) {
+    return HandleFoundFileOrDir(std::move(dir), std::move(filename),
+                                modified_time, size, is_directory);
+  };
+
+  absl::Status status = path::SearchFiles(full_source, recursive_, handler);
+  if (!status.ok()) {
+    return WrapStatus(status,
+                      "Failed to gather source files and directories for '%s'",
+                      full_source);
+  }
+
+  if (files_.size() + dirs_.size() == prev_size) {
+    LOG_WARNING("Neither files nor directories found that match source '%s'",
+                full_source.c_str());
+    // This isn't fatal.
+  }
+
+  return absl::OkStatus();
+}
+
+absl::Status FileFinderAndSender::Flush() {
+  // Flush remaining files.
+  absl::Status status = SendFilesAndDirs();
+  if (!status.ok()) {
+    return WrapStatus(status, "SendFilesAndDirs() failed");
+  }
+
+  // Send an empty batch as EOF indicator.
+  assert(request_.files_size() == 0);
+  status = message_pump_->SendMessage(PacketType::kAddFiles, request_);
+  if (!status.ok()) {
+    return WrapStatus(status, "Failed to send EOF indicator");
+  }
+
+  return absl::OkStatus();
+}
+
+void FileFinderAndSender::ReleaseFiles(std::vector<ClientFileInfo>* files) {
+  *files = std::move(files_);
+}
+
+void FileFinderAndSender::ReleaseDirs(std::vector<ClientDirInfo>* dirs) {
+  *dirs = std::move(dirs_);
+}
+
+absl::Status FileFinderAndSender::HandleFoundFileOrDir(std::string dir,
+                                                       std::string filename,
+                                                       int64_t modified_time,
+                                                       uint64_t size,
+                                                       bool is_directory) {
+  std::string relative_dir = dir.substr(base_dir_.size());
+
+  // Is the path excluded? Check IsEmpty() first to save the path::Join()
+  // if no filter is used (pretty common case).
+  if (!path_filter_->IsEmpty() &&
+      !path_filter_->IsMatch(path::Join(relative_dir, filename))) {
+    return absl::OkStatus();
+  }
+  if (is_directory) {
+    progress_->ReportDirFound();
+  } else {
+    progress_->ReportFileFound();
+  }
+
+  if (request_.directory() != relative_dir) {
+    // Flush files in previous directory.
+    absl::Status status = SendFilesAndDirs();
+    if (!status.ok()) {
+      return WrapStatus(status, "SendFilesAndDirs() failed");
+    }
+
+    // Set new directory.
+    request_.set_directory(relative_dir);
+    request_size_ = request_.directory().length();
+  }
+
+  if (is_directory) {
+    dirs_.emplace_back(path::Join(dir, filename),
+                       static_cast<uint32_t>(base_dir_.size()));
+    request_.add_dirs(filename);
+    request_size_ += filename.size();
+  } else {
+    files_.emplace_back(path::Join(dir, filename), size,
+                        static_cast<uint32_t>(base_dir_.size()));
+
+    AddFilesRequest::File* file = request_.add_files();
+    file->set_filename(filename);
+    file->set_modified_time(modified_time);
+    file->set_size(size);
+    // The serialized proto might have a slightly different length due to
+    // packing, but this doesn't need to be exact.
+    request_size_ += filename.size() + sizeof(modified_time) + sizeof(size);
+  }
+  if (request_size_ >= request_size_threshold_) {
+    absl::Status status = SendFilesAndDirs();
+    if (!status.ok()) {
+      return WrapStatus(status, "SendFilesAndDirs() failed");
+    }
+  }
+
+  return absl::OkStatus();
+}
+
+absl::Status FileFinderAndSender::SendFilesAndDirs() {
+  if (request_.files_size() == 0 && request_.dirs_size() == 0) {
+    return absl::OkStatus();
+  }
+  absl::Status status =
+      message_pump_->SendMessage(PacketType::kAddFiles, request_);
+  if (!status.ok()) {
+    return WrapStatus(status, "Failed to send AddFilesRequest");
+  }
+
+  request_.clear_files();
+  request_.clear_dirs();
+  request_size_ = request_.directory().length();
+  return absl::OkStatus();
+}
+
+}  // namespace cdc_ft

cdc_rsync/file_finder_and_sender.h

@@ -0,0 +1,90 @@
+/*
+ * Copyright 2022 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef CDC_RSYNC_FILE_FINDER_AND_SENDER_H_
+#define CDC_RSYNC_FILE_FINDER_AND_SENDER_H_
+
+#include <string>
+#include <vector>
+
+#include "absl/status/status.h"
+#include "cdc_rsync/client_file_info.h"
+#include "cdc_rsync/protos/messages.pb.h"
+
+namespace cdc_ft {
+
+class MessagePump;
+class PathFilter;
+
+class ReportFindFilesProgress {
+ public:
+  virtual ~ReportFindFilesProgress() = default;
+  virtual void ReportFileFound() = 0;
+  virtual void ReportDirFound() = 0;
+};
+
+class FileFinderAndSender {
+ public:
+  // Send AddFileRequests in packets of roughly 10k max by default.
+  static constexpr size_t kDefaultRequestSizeThreshold = 10000;
+
+  FileFinderAndSender(
+      PathFilter* path_filter, MessagePump* message_pump,
+      ReportFindFilesProgress* progress_, std::string sources_dir,
+      bool recursive, bool relative,
+      size_t request_byte_threshold = kDefaultRequestSizeThreshold);
+  ~FileFinderAndSender();
+
+  absl::Status FindAndSendFiles(std::string source);
+
+  // Sends the remaining file batch to the client, followed by an EOF indicator.
+  // Should be called once all files have been deleted.
+  absl::Status Flush();
+
+  void ReleaseFiles(std::vector<ClientFileInfo>* files);
+  void ReleaseDirs(std::vector<ClientDirInfo>* dirs);
+
+ private:
+  absl::Status HandleFoundFileOrDir(std::string dir, std::string filename,
+                                    int64_t modified_time, uint64_t size,
+                                    bool is_directory);
+
+  // Sends the current file and directory batch to the server.
+  absl::Status SendFilesAndDirs();
+
+  PathFilter* const path_filter_;
+  MessagePump* const message_pump_;
+  ReportFindFilesProgress* const progress_;
+  std::string sources_dir_;
+  const bool recursive_;
+  const bool relative_;
+  const size_t request_size_threshold_;
+
+  // Prefix removed from found files before they are sent to the server.
+  std::string base_dir_;
+  AddFilesRequest request_;
+  size_t request_size_ = 0;
+
+  // Found files.
+  std::vector<ClientFileInfo> files_;
+
+  // Found directories.
+  std::vector<ClientDirInfo> dirs_;
+};
+
+}  // namespace cdc_ft
+
+#endif  // CDC_RSYNC_FILE_FINDER_AND_SENDER_H_

cdc_rsync/file_finder_and_sender_test.cc

@@ -0,0 +1,408 @@
+// Copyright 2022 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "cdc_rsync/file_finder_and_sender.h"
+
+#include <algorithm>
+
+#include "absl/strings/str_format.h"
+#include "absl/strings/str_join.h"
+#include "cdc_rsync/base/fake_socket.h"
+#include "cdc_rsync/base/message_pump.h"
+#include "common/log.h"
+#include "common/path.h"
+#include "common/path_filter.h"
+#include "common/status_test_macros.h"
+#include "common/test_main.h"
+#include "gtest/gtest.h"
+
+namespace cdc_ft {
+namespace {
+
+// Definitions to improve readability.
+constexpr bool kNotRecursive = false;
+constexpr bool kRecursive = true;
+
+constexpr bool kNotRelative = false;
+constexpr bool kRelative = true;
+
+class FakeFindFilesProgress : public ReportFindFilesProgress {
+ public:
+  FakeFindFilesProgress() {}
+  void ReportFileFound() override { num_files_++; }
+  void ReportDirFound() override { num_dirs_++; }
+
+  uint64_t num_files_ = 0;
+  uint64_t num_dirs_ = 0;
+};
+
+class FileFinderAndSenderTest : public ::testing::Test {
+  void SetUp() override {
+    Log::Initialize(std::make_unique<ConsoleLog>(LogLevel::kInfo));
+    message_pump_.StartMessagePump();
+  }
+
+  void TearDown() override {
+    socket_.ShutdownSendingEnd();
+    message_pump_.StopMessagePump();
+    Log::Shutdown();
+  }
+
+ protected:
+  struct ReceivedFile {
+    std::string dir;
+    std::string file;
+    ReceivedFile(std::string dir, std::string file)
+        : dir(std::move(dir)), file(std::move(file)) {}
+  };
+
+  struct ReceivedFileFormatter {
+    void operator()(std::string* out, const ReceivedFile& val) const {
+      absl::StrAppend(out, absl::StrFormat("{ %s, %s }", val.dir, val.file));
+    }
+  };
+
+  void ExpectReceiveFiles(std::vector<ReceivedFile> expected_data,
+                          std::vector<int> expected_batch_count = {}) {
+    std::vector<ReceivedFile> data;
+    std::vector<int> batch_count;
+    AddFilesRequest request;
+    for (;;) {
+      EXPECT_OK(message_pump_.ReceiveMessage(PacketType::kAddFiles, &request));
+      if (request.files_size() == 0 && request.dirs_size() == 0) {
+        // EOF.
+        break;
+      }
+
+      batch_count.push_back(request.files_size() + request.dirs_size());
+      for (const auto& file : request.files()) {
+        data.emplace_back(request.directory(), file.filename());
+      }
+      for (const auto& dir : request.dirs()) {
+        data.emplace_back(request.directory(), dir);
+      }
+    }
+
+    // expected_batch_count can be empty for convenience.
+    if (!expected_batch_count.empty()) {
+      EXPECT_EQ(absl::StrJoin(batch_count, ", "),
+                absl::StrJoin(expected_batch_count, ", "));
+    }
+
+    EXPECT_EQ(absl::StrJoin(data, ", ", ReceivedFileFormatter()),
+              absl::StrJoin(expected_data, ", ", ReceivedFileFormatter()));
+  }
+
+  FakeSocket socket_;
+  FakeFindFilesProgress progress_;
+  PathFilter path_filter_;
+  MessagePump message_pump_{&socket_, MessagePump::PacketReceivedDelegate()};
+
+  std::string base_dir_ = GetTestDataDir("file_finder_and_sender");
+};
+
+TEST_F(FileFinderAndSenderTest, FindNonRecursive) {
+  FileFinderAndSender finder(&path_filter_, &message_pump_, &progress_, "",
+                             kNotRecursive, kNotRelative);
+
+  EXPECT_OK(finder.FindAndSendFiles(base_dir_));
+  EXPECT_EQ(progress_.num_files_, 0);
+  std::vector<ClientFileInfo> files;
+  EXPECT_OK(finder.Flush());
+  finder.ReleaseFiles(&files);
+
+  ASSERT_TRUE(files.empty());
+  ExpectReceiveFiles({{}});
+}
+
+TEST_F(FileFinderAndSenderTest, FindRecursive) {
+  FileFinderAndSender finder(&path_filter_, &message_pump_, &progress_, "",
+                             kRecursive, kNotRelative);
+
+  EXPECT_OK(finder.FindAndSendFiles(base_dir_));
+  EXPECT_EQ(progress_.num_files_, 5);
+  EXPECT_EQ(progress_.num_dirs_, 2);
+
+  EXPECT_OK(finder.Flush());
+  std::vector<ClientFileInfo> files;
+  finder.ReleaseFiles(&files);
+  std::vector<ClientDirInfo> dirs;
+  finder.ReleaseDirs(&dirs);
+
+  ASSERT_EQ(files.size(), 5);
+  EXPECT_EQ(files[0].path, path::Join(base_dir_, "a.txt"));
+  EXPECT_EQ(files[1].path, path::Join(base_dir_, "b.txt"));
+  EXPECT_EQ(files[2].path, path::Join(base_dir_, "c.txt"));
+  EXPECT_EQ(files[3].path, path::Join(base_dir_, "subdir", "d.txt"));
+  EXPECT_EQ(files[4].path, path::Join(base_dir_, "subdir", "e.txt"));
+
+  ASSERT_EQ(dirs.size(), 2);
+  EXPECT_EQ(dirs[0].path, base_dir_);
+  EXPECT_EQ(dirs[1].path, path::Join(base_dir_, "subdir"));
+
+  // Verify that the data sent to the socket matches.
+  ExpectReceiveFiles({{"", "file_finder_and_sender"},
+                      {"file_finder_and_sender\\", "a.txt"},
+                      {"file_finder_and_sender\\", "b.txt"},
+                      {"file_finder_and_sender\\", "c.txt"},
+                      {"file_finder_and_sender\\", "subdir"},
+                      {"file_finder_and_sender\\subdir\\", "d.txt"},
+                      {"file_finder_and_sender\\subdir\\", "e.txt"}},
+                     {1, 4, 2});
+}
+
+TEST_F(FileFinderAndSenderTest, FindWithSmallerBatchSize) {
+  // Tweak size threshold so that we get 2 files per batch for the base dir.
+  // This tests that the batch gets flushed when the directory changes since
+  // the base dir has 3 files.
+  int request_byte_size_threshold = static_cast<int>(
+      strlen("file_finder_and_sender\\") +  // directory
+      sizeof(int64_t) + sizeof(uint64_t) +  // modified_time + size
+      strlen("a.txt") + 3);                 // filename + some slack
+  FileFinderAndSender finder(&path_filter_, &message_pump_, &progress_, "",
+                             kRecursive, kNotRelative,
+                             request_byte_size_threshold);
+
+  EXPECT_OK(finder.FindAndSendFiles(base_dir_));
+  EXPECT_OK(finder.Flush());
+  EXPECT_EQ(progress_.num_files_, 5);
+  ASSERT_EQ(progress_.num_dirs_, 2);
+
+  // Note that the expected batch size is {1, 2, 2, 1, 1} here due to the
+  // smaller request sizes.
+  ExpectReceiveFiles({{"", "file_finder_and_sender"},
+                      {"file_finder_and_sender\\", "a.txt"},
+                      {"file_finder_and_sender\\", "b.txt"},
+                      {"file_finder_and_sender\\", "c.txt"},
+                      {"file_finder_and_sender\\", "subdir"},
+                      {"file_finder_and_sender\\subdir\\", "d.txt"},
+                      {"file_finder_and_sender\\subdir\\", "e.txt"}},
+                     {1, 2, 2, 1, 1});
+}
+
+TEST_F(FileFinderAndSenderTest, FindWithFilter) {
+  path_filter_.AddRule(PathFilter::Rule::Type::kExclude, "*b.txt");
+  FileFinderAndSender finder(&path_filter_, &message_pump_, &progress_, "",
+                             kNotRecursive, kNotRelative);
+
+  EXPECT_OK(finder.FindAndSendFiles(path::Join(base_dir_, "*")));
+  EXPECT_EQ(progress_.num_files_, 2);
+  EXPECT_EQ(progress_.num_dirs_, 1);
+
+  EXPECT_OK(finder.Flush());
+  std::vector<ClientFileInfo> files;
+  finder.ReleaseFiles(&files);
+  std::vector<ClientDirInfo> dirs;
+  finder.ReleaseDirs(&dirs);
+
+  ASSERT_EQ(files.size(), 2);
+  EXPECT_EQ(files[0].path, path::Join(base_dir_, "a.txt"));
+  EXPECT_EQ(files[1].path, path::Join(base_dir_, "c.txt"));
+
+  ASSERT_EQ(dirs.size(), 1);
+  EXPECT_EQ(dirs[0].path, path::Join(base_dir_, "subdir"));
+
+  ExpectReceiveFiles({{"", "a.txt"}, {"", "c.txt"}, {"", "subdir"}});
+}
+
+TEST_F(FileFinderAndSenderTest, FindWithDot) {
+  FileFinderAndSender finder(&path_filter_, &message_pump_, &progress_, "",
+                             kRecursive, kNotRelative);
+
+  EXPECT_OK(finder.FindAndSendFiles(base_dir_ + "\\."));
+  EXPECT_EQ(progress_.num_files_, 5);
+  EXPECT_EQ(progress_.num_dirs_, 1);
+
+  EXPECT_OK(finder.Flush());
+
+  ExpectReceiveFiles({{"", "a.txt"},
+                      {"", "b.txt"},
+                      {"", "c.txt"},
+                      {"", "subdir"},
+                      {"subdir\\", "d.txt"},
+                      {"subdir\\", "e.txt"}},
+                     {});
+}
+
+TEST_F(FileFinderAndSenderTest, FindWithForwardSlash) {
+  FileFinderAndSender finder(&path_filter_, &message_pump_, &progress_, "",
+                             kRecursive, kNotRelative);
+
+  std::string base_dir_forward = GetTestDataDir("file_finder_and_sender");
+  std::replace(base_dir_forward.begin(), base_dir_forward.end(), '\\', '/');
+
+  EXPECT_OK(finder.FindAndSendFiles(base_dir_forward + "/."));
+  EXPECT_EQ(progress_.num_files_, 5);
+  EXPECT_EQ(progress_.num_dirs_, 1);
+
+  EXPECT_OK(finder.Flush());
+
+  ExpectReceiveFiles({{"", "a.txt"},
+                      {"", "b.txt"},
+                      {"", "c.txt"},
+                      {"", "subdir"},
+                      {"subdir\\", "d.txt"},
+                      {"subdir\\", "e.txt"}},
+                     {});
+}
+
+TEST_F(FileFinderAndSenderTest, FindWithRelative) {
+  FileFinderAndSender finder(&path_filter_, &message_pump_, &progress_, "",
+                             kNotRecursive, kRelative);
+
+  std::vector<std::string> sources = {
+      path::Join(base_dir_, "a.txt"), path::Join(base_dir_, ".", "b.txt"),
+      path::Join(base_dir_, ".", "subdir", "d.txt"),
+      path::Join(base_dir_, "subdir", ".", "e.txt")};
+
+  for (const std::string& source : sources) {
+    EXPECT_OK(finder.FindAndSendFiles(source));
+  }
+
+  EXPECT_EQ(progress_.num_files_, 4);
+  std::vector<ClientFileInfo> files;
+  EXPECT_OK(finder.Flush());
+  finder.ReleaseFiles(&files);
+
+  ASSERT_EQ(files.size(), 4);
+  EXPECT_EQ(files[0].path, sources[0]);
+  EXPECT_EQ(files[1].path, sources[1]);
+  EXPECT_EQ(files[2].path, sources[2]);
+  EXPECT_EQ(files[3].path, sources[3]);
+
+  // The paths sent to the socket should have the correct relative paths.
+  std::string rel =
+      base_dir_.substr(path::GetDrivePrefix(base_dir_).size() + 1) + "\\";
+
+  ExpectReceiveFiles(
+      {{rel, "a.txt"}, {"", "b.txt"}, {"subdir\\", "d.txt"}, {"", "e.txt"}});
+}
+
+TEST_F(FileFinderAndSenderTest, FindWithRelativeAndSourcesDir) {
+  // Just go to the parent directory, we just need some existing dir.
+  std::string sources_dir = path::DirName(base_dir_);
+  path::EnsureEndsWithPathSeparator(&sources_dir);
+  std::string rel_dir = base_dir_.substr(sources_dir.size());
+
+  FileFinderAndSender finder(&path_filter_, &message_pump_, &progress_,
+                             sources_dir, kNotRecursive, kRelative);
+
+  std::vector<std::string> sources = {
+      path::Join(rel_dir, "a.txt"), path::Join(rel_dir, ".", "b.txt"),
+      path::Join(rel_dir, "subdir", "d.txt"),
+      path::Join(rel_dir, "subdir", ".", "e.txt")};
+
+  for (const std::string& source : sources) {
+    EXPECT_OK(finder.FindAndSendFiles(source));
+  }
+
+  EXPECT_EQ(progress_.num_files_, 4);
+  std::vector<ClientFileInfo> files;
+  EXPECT_OK(finder.Flush());
+  finder.ReleaseFiles(&files);
+
+  ASSERT_EQ(files.size(), 4);
+  EXPECT_EQ(files[0].path, path::Join(sources_dir, sources[0]));
+  EXPECT_EQ(files[1].path, path::Join(sources_dir, sources[1]));
+  EXPECT_EQ(files[2].path, path::Join(sources_dir, sources[2]));
+  EXPECT_EQ(files[3].path, path::Join(sources_dir, sources[3]));
+
+  path::EnsureEndsWithPathSeparator(&rel_dir);
+  ExpectReceiveFiles({{rel_dir, "a.txt"},
+                      {"", "b.txt"},
+                      {rel_dir + "subdir\\", "d.txt"},
+                      {"", "e.txt"}});
+}
+
+TEST_F(FileFinderAndSenderTest,
+       FindWithRelativeAndSourcesDirForwardSlashInSouceDir) {
+  // Just go to the parent directory, we just need some existing dir.
+  std::string sources_dir = path::DirName(base_dir_);
+  path::EnsureEndsWithPathSeparator(&sources_dir);
+  std::string rel_dir = base_dir_.substr(sources_dir.size());
+
+  std::string sources_dir_forward(sources_dir);
+  std::replace(sources_dir_forward.begin(), sources_dir_forward.end(), '\\',
+               '/');
+  FileFinderAndSender finder(&path_filter_, &message_pump_, &progress_,
+                             sources_dir_forward, kNotRecursive, kRelative);
+
+  std::vector<std::string> sources = {
+      path::Join(rel_dir, "a.txt"), path::Join(rel_dir, ".", "b.txt"),
+      path::Join(rel_dir, "subdir", "d.txt"),
+      path::Join(rel_dir, "subdir", ".", "e.txt")};
+
+  for (const std::string& source : sources) {
+    EXPECT_OK(finder.FindAndSendFiles(source));
+  }
+
+  EXPECT_EQ(progress_.num_files_, 4);
+  std::vector<ClientFileInfo> files;
+  EXPECT_OK(finder.Flush());
+  finder.ReleaseFiles(&files);
+
+  ASSERT_EQ(files.size(), 4);
+  EXPECT_EQ(files[0].path, path::Join(sources_dir, sources[0]));
+  EXPECT_EQ(files[1].path, path::Join(sources_dir, sources[1]));
+  EXPECT_EQ(files[2].path, path::Join(sources_dir, sources[2]));
+  EXPECT_EQ(files[3].path, path::Join(sources_dir, sources[3]));
+
+  path::EnsureEndsWithPathSeparator(&rel_dir);
+  ExpectReceiveFiles({{rel_dir, "a.txt"},
+                      {"", "b.txt"},
+                      {rel_dir + "subdir\\", "d.txt"},
+                      {"", "e.txt"}});
+}
+TEST_F(FileFinderAndSenderTest,
+       FindWithRelativeAndSourcesDirForwardSlashInSourceDirFiles) {
+  // Just go to the parent directory, we just need some existing dir.
+  std::string sources_dir = path::DirName(base_dir_);
+  path::EnsureEndsWithPathSeparator(&sources_dir);
+  std::string rel_dir = base_dir_.substr(sources_dir.size());
+
+  std::string sources_dir_forward(sources_dir);
+  std::replace(sources_dir_forward.begin(), sources_dir_forward.end(), '\\',
+               '/');
+  FileFinderAndSender finder(&path_filter_, &message_pump_, &progress_,
+                             sources_dir_forward, kNotRecursive, kRelative);
+
+  std::vector<std::string> sources = {rel_dir + "/a.txt", rel_dir + "/./b.txt",
+                                      rel_dir + "/subdir/d.txt",
+                                      rel_dir + "/subdir/./e.txt"};
+
+  for (const std::string& source : sources) {
+    EXPECT_OK(finder.FindAndSendFiles(source));
+  }
+
+  EXPECT_EQ(progress_.num_files_, 4);
+  std::vector<ClientFileInfo> files;
+  EXPECT_OK(finder.Flush());
+  finder.ReleaseFiles(&files);
+
+  ASSERT_EQ(files.size(), 4);
+  for (std::string& source : sources) path::FixPathSeparators(&source);
+  EXPECT_EQ(files[0].path, path::Join(sources_dir, sources[0]));
+  EXPECT_EQ(files[1].path, path::Join(sources_dir, sources[1]));
+  EXPECT_EQ(files[2].path, path::Join(sources_dir, sources[2]));
+  EXPECT_EQ(files[3].path, path::Join(sources_dir, sources[3]));
+
+  path::EnsureEndsWithPathSeparator(&rel_dir);
+  ExpectReceiveFiles({{rel_dir, "a.txt"},
+                      {"", "b.txt"},
+                      {rel_dir + "subdir\\", "d.txt"},
+                      {"", "e.txt"}});
+}
+
+}  // namespace
+}  // namespace cdc_ft

cdc_rsync/parallel_file_opener.cc

@@ -0,0 +1,122 @@
+// Copyright 2022 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "cdc_rsync/parallel_file_opener.h"
+
+#include "absl/status/statusor.h"
+#include "common/path.h"
+
+namespace cdc_ft {
+namespace {
+
+// Number of threads in the pool.
+size_t GetPoolSize() {
+  uint32_t num_threads = std::thread::hardware_concurrency();
+  if (num_threads == 0) return 4;
+  return num_threads;
+}
+
+// Number of file open operations to queue in advance.
+const size_t kNumQueuedTasks = 256;
+
+}  // namespace
+
+namespace internal {
+
+class FileOpenTask : public Task {
+ public:
+  FileOpenTask(size_t index, ClientFileInfo file)
+      : index_(index), file_(file) {}
+
+  ~FileOpenTask() {
+    if (*fp_) {
+      fclose(*fp_);
+      *fp_ = nullptr;
+    }
+  }
+
+  FileOpenTask(const FileOpenTask& other) = delete;
+  FileOpenTask(const FileOpenTask&& other) = delete;
+
+  FileOpenTask& operator=(FileOpenTask&) = delete;
+  FileOpenTask& operator=(FileOpenTask&&) = delete;
+
+  void ThreadRun(IsCancelledPredicate is_cancelled) override {
+    fp_ = path::OpenFile(file_.path, "rb");
+  }
+
+  size_t Index() const { return index_; }
+
+  FILE* ReleaseFile() {
+    FILE* fp = *fp_;
+    *fp_ = nullptr;
+    return fp;
+  }
+
+ private:
+  size_t index_;
+  ClientFileInfo file_;
+  absl::StatusOr<FILE*> fp_ = nullptr;
+};
+
+}  // namespace internal
+
+ParallelFileOpener::ParallelFileOpener(
+    const std::vector<ClientFileInfo>* files,
+    const std::vector<uint32_t>& file_indices)
+    : files_(files), file_indices_(file_indices), pool_(GetPoolSize()) {
+  // Queue the first |kNumQueuedTasks| files (if available).
+  size_t num_to_queue = std::min(kNumQueuedTasks, file_indices_.size());
+  for (size_t n = 0; n < num_to_queue; ++n) {
+    QueueNextFile();
+  }
+}
+
+ParallelFileOpener::~ParallelFileOpener() = default;
+
+FILE* ParallelFileOpener::GetNextOpenFile() {
+  if (curr_index_ >= file_indices_.size()) {
+    return nullptr;
+  }
+
+  QueueNextFile();
+
+  // Wait until the file at |curr_index_| is available.
+  // Note that |index_to_completed_tasks_| is sorted by index.
+  while (index_to_completed_tasks_.empty() ||
+         index_to_completed_tasks_.begin()->first != curr_index_) {
+    std::unique_ptr<Task> task = pool_.GetCompletedTask();
+    auto* fopen_task = static_cast<internal::FileOpenTask*>(task.release());
+    index_to_completed_tasks_[fopen_task->Index()].reset(fopen_task);
+  }
+
+  // The first completed task should be the one for |curr_index_|.
+  const auto& first_iter = index_to_completed_tasks_.begin();
+  FILE* file = first_iter->second->ReleaseFile();
+  index_to_completed_tasks_.erase(first_iter);
+  curr_index_++;
+  return file;
+}
+
+void ParallelFileOpener::QueueNextFile() {
+  if (look_ahead_index_ >= file_indices_.size()) {
+    return;
+  }
+
+  pool_.QueueTask(std::make_unique<internal::FileOpenTask>(
+      look_ahead_index_, files_->at(file_indices_[look_ahead_index_])));
+  ++look_ahead_index_;
+}
+
+}  // namespace cdc_ft

cdc_rsync/parallel_file_opener.h

@@ -0,0 +1,74 @@
+/*
+ * Copyright 2022 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef CDC_RSYNC_PARALLEL_FILE_OPENER_H_
+#define CDC_RSYNC_PARALLEL_FILE_OPENER_H_
+
+#include <map>
+#include <memory>
+#include <vector>
+
+#include "cdc_rsync/client_file_info.h"
+#include "common/threadpool.h"
+
+namespace cdc_ft {
+
+namespace internal {
+class FileOpenTask;
+}
+
+// Opens files on a background worker thread pool. This improves performance in
+// cases where many files have to be opened quickly.
+class ParallelFileOpener {
+ public:
+  // Starts opening the |files| indexed by |file_indices| in the background.
+  ParallelFileOpener(const std::vector<ClientFileInfo>* files,
+                     const std::vector<uint32_t>& file_indices);
+
+  ~ParallelFileOpener();
+
+  // Returns FILE* pointer from opening a file from |files| in rb-mode.
+  // The first call returns the FILE* pointer for files[file_indices[0]],
+  // the second call returns the FILE* pointer for files[file_indices[1]] etc.
+  // The caller must close the file.
+  FILE* GetNextOpenFile();
+
+ private:
+  // Queues another open file task.
+  void QueueNextFile();
+
+  // Pointer to list of files, not owned.
+  const std::vector<ClientFileInfo>* files_;
+
+  // Indices into the |files_| to open.
+  std::vector<uint32_t> file_indices_;
+
+  // Index into |file_indices_| of the next file returned by GetNextOpenFile().
+  size_t curr_index_ = 0;
+
+  // Index into |file_indices_| of the file queued by QueueNextFile().
+  size_t look_ahead_index_ = 0;
+
+  // Maps index into |file_indices_| to completed task.
+  std::map<size_t, std::unique_ptr<internal::FileOpenTask>>
+      index_to_completed_tasks_;
+
+  Threadpool pool_;
+};
+
+}  // namespace cdc_ft
+
+#endif  // CDC_RSYNC_PARALLEL_FILE_OPENER_H_

cdc_rsync/parallel_file_opener_test.cc

@@ -0,0 +1,78 @@
+// Copyright 2022 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "cdc_rsync/parallel_file_opener.h"
+
+#include "common/path.h"
+#include "common/test_main.h"
+#include "gtest/gtest.h"
+
+namespace cdc_ft {
+namespace {
+
+class ParallelFileOpenerTest : public ::testing::Test {
+ protected:
+  std::string base_dir_ = GetTestDataDir("parallel_file_opener");
+
+  // Args 2 (file size) and 3 (base dir len) are not used.
+  std::vector<ClientFileInfo> files_ = {
+      ClientFileInfo(path::Join(base_dir_, "file1.txt"), 0, 0),
+      ClientFileInfo(path::Join(base_dir_, "file2.txt"), 0, 0),
+      ClientFileInfo(path::Join(base_dir_, "file3.txt"), 0, 0),
+  };
+
+  std::string ReadAndClose(FILE* file) {
+    char line[256] = {0};
+    EXPECT_TRUE(fgets(line, sizeof(line) - 1, file));
+    fclose(file);
+    return line;
+  }
+};
+
+TEST_F(ParallelFileOpenerTest, OpenNoFiles) {
+  ParallelFileOpener file_opener(&files_, {});
+  EXPECT_EQ(file_opener.GetNextOpenFile(), nullptr);
+}
+
+TEST_F(ParallelFileOpenerTest, OpenSingleFile) {
+  ASSERT_GE(files_.size(), 3);
+  ParallelFileOpener file_opener(&files_, {1});
+
+  FILE* file2 = file_opener.GetNextOpenFile();
+  ASSERT_NE(file2, nullptr);
+  EXPECT_EQ(ReadAndClose(file2), "data2");
+
+  EXPECT_EQ(file_opener.GetNextOpenFile(), nullptr);
+}
+
+TEST_F(ParallelFileOpenerTest, OpenManyFiles) {
+  const int num_indices = 500;
+  std::vector<uint32_t> indices;
+  for (int n = 0; n < num_indices; ++n) {
+    indices.push_back(n % files_.size());
+  }
+
+  ParallelFileOpener file_opener(&files_, indices);
+
+  for (int n = 0; n < num_indices; ++n) {
+    FILE* file = file_opener.GetNextOpenFile();
+    ASSERT_NE(file, nullptr);
+    EXPECT_EQ(ReadAndClose(file), "data" + std::to_string(indices[n] + 1));
+  }
+
+  EXPECT_EQ(file_opener.GetNextOpenFile(), nullptr);
+}
+
+}  // namespace
+}  // namespace cdc_ft

cdc_rsync/progress_tracker.cc

@@ -0,0 +1,549 @@
+// Copyright 2022 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "cdc_rsync/progress_tracker.h"
+
+#include <algorithm>
+#include <cassert>
+
+#include "absl/strings/str_format.h"
+#include "common/util.h"
+#include "json/json.h"
+
+namespace cdc_ft {
+
+namespace {
+
+// Count signature progress 1/40 because it's probably faster than the rest.
+// This assumes a sig speed of 800 MB/sec and a diffing speed of 20 MB/sec,
+// but since we don't know the exact numbers, we're estimating them.
+constexpr int kSigFactor = 40;
+
+// Fills up |str| with spaces up to a string length of |size|.
+void PaddRight(std::string* str, size_t size) {
+  if (str->size() < size) {
+    str->insert(str->size(), size - str->size(), ' ');
+  }
+}
+
+// Shortens |filepath| if it is longer than |max_len|.
+// TODO: Improve this, e.g. by replacing directories in the middle by "..".
+// TODO: Also make sure it plays nicely with UTF-8 code points.
+std::string ShortenFilePath(const std::string filepath, size_t max_len) {
+  if (filepath.size() > max_len && max_len >= 3) {
+    return "..." + filepath.substr(filepath.size() - max_len + 3);
+  }
+  return filepath;
+}
+
+// Formatting might be messed up for >9999 ZB. Please don't sync large files!
+const char* kSizeUnits[] = {"B", "KB", "MB", "GB", "TB", "PB", "EB", "ZB"};
+
+// Divides |size| by 1024 as long as it has more than 4 digits and returns the
+// corresponding unit, e.g. 10240 -> 10KB.
+const char* FormatIntBytes(uint64_t* size) {
+  int unitIdx = 0;
+
+  while (*size > 9999 && unitIdx < std::size(kSizeUnits) - 1) {
+    ++unitIdx;
+    *size /= 1024;
+  }
+
+  return kSizeUnits[unitIdx];
+}
+
+// Divides |size| by 1024 as long as it has more than 3 digits and returns the
+// corresponding unit, e.g. 1500 -> 1.5KB.
+const char* FormatDoubleBytes(double* size) {
+  int unitIdx = 0;
+
+  while (*size > 999.9 && unitIdx < std::size(kSizeUnits) - 1) {
+    ++unitIdx;
+    *size /= 1024.0;
+  }
+
+  return kSizeUnits[unitIdx];
+}
+
+// Formats |sec| seconds into hh::mm:ss if more than one hour or else mm:ss.
+std::string FormatTime(double sec) {
+  int isec = static_cast<int>(sec);
+  int ihour = isec / 3600;
+  isec -= ihour * 3600;
+  int imin = isec / 60;
+  isec -= imin * 60;
+
+  if (ihour > 0) {
+    return absl::StrFormat("%02i:%02i:%02i", ihour, imin, isec);
+  }
+
+  return absl::StrFormat("%02i:%02i", imin, isec);
+}
+
+// Returns curr/total as double number, clamped to [0,1].
+double GetProgress(uint64_t curr, uint64_t total) {
+  double progress = static_cast<double>(curr) / std::max<uint64_t>(1, total);
+  return std::min(std::max(progress, 0.0), 1.0);
+}
+
+}  // namespace
+
+ConsoleProgressPrinter::ConsoleProgressPrinter(bool quiet, bool is_tty)
+    : ProgressPrinter(quiet, is_tty) {}
+
+void ConsoleProgressPrinter::Print(std::string text, bool newline,
+                                   int output_width) {
+  if (quiet()) {
+    return;
+  }
+
+  char linechar = newline || !is_tty() ? '\n' : '\r';
+  if (is_tty()) PaddRight(&text, output_width);
+  printf("%s%c", text.c_str(), linechar);
+  if (!is_tty()) fflush(stdout);
+}
+
+ProgressTracker::ProgressTracker(ProgressPrinter* printer, int verbosity,
+                                 bool json, int fixed_output_width,
+                                 SteadyClock* clock)
+    : printer_(printer),
+      only_total_progress_(verbosity == 0),
+      json_(json),
+      fixed_output_width_(fixed_output_width),
+      display_delay_sec_(printer_->is_tty() ? 0.1 : 1.0),
+      total_timer_(clock),
+      file_timer_(clock),
+      sig_timer_(clock),
+      print_timer_(clock) {}
+
+ProgressTracker::~ProgressTracker() = default;
+
+void ProgressTracker::StartFindFiles() {
+  assert(state_ == State::kIdle);
+  state_ = State::kSearch;
+
+  files_found_ = 0;
+}
+
+void ProgressTracker::ReportFileFound() {
+  assert(state_ == State::kSearch);
+  ++files_found_;
+
+  UpdateOutput(false);
+}
+
+void ProgressTracker::ReportDirFound() {
+  assert(state_ == State::kSearch);
+  ++dirs_found_;
+
+  UpdateOutput(false);
+}
+
+void ProgressTracker::ReportFileStats(
+    uint32_t num_missing_files, uint32_t num_extraneous_files,
+    uint32_t num_matching_files, uint32_t num_changed_files,
+    uint64_t total_missing_bytes, uint64_t total_changed_client_bytes,
+    uint64_t total_changed_server_bytes, uint32_t num_missing_dirs,
+    uint32_t num_extraneous_dirs, uint32_t num_matching_dirs,
+    bool whole_file_arg, bool checksum_arg, bool delete_arg) {
+  const char* fmt[] = {
+      "%6u file(s) and %u folder(s) are not present on the instance and will "
+      "be copied.",
+      "%6u file(s) changed and will be updated.",
+      "%6u file(s) and %u folder(s) match and do not have to be updated.",
+      "%6u file(s) and %u folder(s) on the instance do not exist on this "
+      "machine."};
+
+  if (whole_file_arg) {
+    fmt[1] = "%6u file(s) changed and will be copied due to -W/--whole-file.";
+  }
+
+  if (checksum_arg) {
+    fmt[2] =
+        "%6u file(s) and %u folder(s) have matching modified time and size, "
+        "but will be synced due to -c/--checksum.";
+  }
+
+  if (checksum_arg & whole_file_arg) {
+    fmt[2] =
+        "%6u file(s) and %u folder(s) have matching modified time and size, "
+        "but will be copied due to -c/--checksum and -W/--whole-file.";
+  }
+
+  if (delete_arg) {
+    fmt[3] =
+        "%6u file(s) and %u folder(s) on the instance do not exist on this "
+        "machine and will be deleted due to --delete.";
+  }
+
+  Print(absl::StrFormat(fmt[0], num_missing_files, num_missing_dirs), true);
+  Print(absl::StrFormat(fmt[1], num_changed_files), true);
+  Print(absl::StrFormat(fmt[2], num_matching_files, num_matching_dirs), true);
+  Print(absl::StrFormat(fmt[3], num_extraneous_files, num_extraneous_dirs),
+        true);
+
+  total_bytes_to_copy_ = total_missing_bytes;
+  total_bytes_to_diff_ = total_changed_client_bytes;
+  total_sig_bytes_ = total_changed_server_bytes;
+  total_files_to_delete_ = num_extraneous_files;
+  total_dirs_to_delete_ = num_extraneous_dirs;
+}
+
+void ProgressTracker::StartCopy(const std::string& filepath,
+                                uint64_t filesize) {
+  assert(state_ == State::kIdle);
+  state_ = State::kCopy;
+  file_timer_.Reset();
+  sig_time_sec_ = 0;
+
+  curr_filepath_ = filepath;
+  curr_filesize_ = filesize;
+  curr_bytes_copied_ = 0;
+}
+
+void ProgressTracker::ReportCopyProgress(uint64_t num_bytes_copied) {
+  assert(state_ == State::kCopy);
+  curr_bytes_copied_ += num_bytes_copied;
+  total_bytes_copied_ += num_bytes_copied;
+
+  UpdateOutput(false);
+}
+
+void ProgressTracker::StartSync(const std::string& filepath,
+                                uint64_t client_size, uint64_t server_size) {
+  assert(state_ == State::kIdle);
+  state_ = State::kSyncDiff;
+  file_timer_.Reset();
+  sig_timer_.Reset();
+  sig_time_sec_ = 0;
+
+  curr_filepath_ = filepath;
+  curr_filesize_ = client_size;
+  server_filesize_ = server_size;
+  curr_sig_bytes_read_ = 0;
+  curr_bytes_diffed_ = 0;
+}
+
+void ProgressTracker::ReportSyncProgress(size_t num_client_bytes_processed,
+                                         size_t num_server_bytes_processed) {
+  assert(state_ == State::kSyncSig || state_ == State::kSyncDiff);
+
+  // If diffing is blocked on getting more server chunks, switch to kSyncSig,
+  // which effectively changes the output from "Dxxx%" to "Sxxx%" and removes
+  // the ETA.
+  State new_state = state_;
+  if (num_client_bytes_processed > 0) {
+    new_state = State::kSyncDiff;
+  } else if (num_server_bytes_processed > 0) {
+    new_state = State::kSyncSig;
+  }
+
+  // Measure time exclusively spent in server-side signature computation. This
+  // is later taken into account to get a better speed estimation for diffing.
+  if (state_ == State::kSyncDiff && new_state == State::kSyncSig) {
+    sig_timer_.Reset();
+  } else if (state_ == State::kSyncSig && new_state == State::kSyncDiff) {
+    sig_time_sec_ += sig_timer_.ElapsedSeconds();
+  }
+  state_ = new_state;
+
+  curr_bytes_diffed_ += num_client_bytes_processed;
+  total_bytes_diffed_ += num_client_bytes_processed;
+  curr_sig_bytes_read_ += num_server_bytes_processed;
+  total_sig_bytes_read_ += num_server_bytes_processed;
+
+  UpdateOutput(false);
+}
+
+void ProgressTracker::StartDeleteFiles() {
+  assert(state_ == State::kIdle);
+  state_ = State::kDelete;
+
+  files_deleted_ = 0;
+}
+
+void ProgressTracker::ReportFileDeleted(const std::string& filepath) {
+  curr_filepath_ = filepath;
+  ++files_deleted_;
+
+  UpdateOutput(false);
+}
+
+void ProgressTracker::ReportDirDeleted(const std::string& filepath) {
+  curr_filepath_ = filepath;
+  ++dirs_deleted_;
+
+  UpdateOutput(false);
+}
+
+void ProgressTracker::Finish() {
+  assert(state_ != State::kIdle);
+
+  UpdateOutput(true);
+  if (state_ == State::kSearch) {
+    // Total time does not count file search time.
+    total_timer_.Reset();
+  }
+
+  state_ = State::kIdle;
+}
+
+void ProgressTracker::UpdateOutput(bool finished) {
+  if (printer_->quiet()) {
+    return;
+  }
+
+  if (state_ == State::kDelete) {
+    if (total_files_to_delete_ + total_dirs_to_delete_ == 0 ||
+        (!only_total_progress_ && finished)) {
+      // No need to print here, it would result in "0/0" or duplicate lines.
+      return;
+    }
+
+    if (only_total_progress_) {
+      if (!finished && print_timer_.ElapsedSeconds() < display_delay_sec_) {
+        return;
+      }
+      print_timer_.Reset();
+
+      Print(absl::StrFormat("%u/%u file(s) and %u/%u folder(s) deleted.",
+                            files_deleted_, total_files_to_delete_,
+                            dirs_deleted_, total_dirs_to_delete_),
+            finished);
+      return;
+    }
+
+    std::string txt =
+        absl::StrFormat("deleted %u / %u", files_deleted_ + dirs_deleted_,
+                        total_files_to_delete_ + total_dirs_to_delete_);
+
+    int width = GetOutputWidth();
+    int file_width = std::max(12, width - static_cast<int>(txt.size()));
+    std::string short_path = ShortenFilePath(curr_filepath_, file_width);
+    PaddRight(&short_path, file_width);
+    printer_->Print(short_path + txt, true, width);
+    return;
+  }
+
+  if (only_total_progress_ && state_ != State::kSearch) {
+    finished &= GetTotalProgress() == 1;
+    if (!finished && print_timer_.ElapsedSeconds() < display_delay_sec_) {
+      return;
+    }
+    print_timer_.Reset();
+
+    if (json_) {
+      double total_progress, total_sec, total_eta_sec;
+      GetTotalProgressStats(&total_progress, &total_sec, &total_eta_sec);
+
+      Json::Value val;
+      val["total_progress"] = total_progress;
+      val["total_duration"] = total_sec;
+      val["total_eta"] = total_eta_sec;
+      PrintJson(val, finished);
+    } else {
+      Print(GetTotalProgressText(), finished);
+    }
+    return;
+  }
+
+  // Always print if finished (to make sure to get the line feed).
+  if (!finished && print_timer_.ElapsedSeconds() < display_delay_sec_) {
+    return;
+  }
+  print_timer_.Reset();
+
+  switch (state_) {
+    case State::kSearch: {
+      Print(absl::StrFormat("%u file(s) and %u folder(s) found", files_found_,
+                            dirs_found_),
+            finished);
+      break;
+    }
+
+    case State::kCopy: {
+      // file               C 50% 12345MB 123.4MB/s 00:10 ETA  50% TOT 05:00 ETA
+      double progress = GetProgress(curr_bytes_copied_, curr_filesize_);
+      OutputFileProgress(OutputType::kCopy, progress, finished);
+      break;
+    }
+
+    case State::kSyncSig: {
+      // file               S 50% 12345MB ---.-MB/s 00:10 ETA  50% TOT 05:00 ETA
+      double progress =
+          GetProgress(curr_bytes_diffed_ + curr_sig_bytes_read_ / kSigFactor,
+                      curr_filesize_ + server_filesize_ / kSigFactor);
+      OutputFileProgress(OutputType::kSig, progress, finished);
+      break;
+    }
+
+    case State::kSyncDiff: {
+      // file               D 50% 12345MB 123.4MB/s 00:10 ETA  50% TOT 05:00 ETA
+      double progress =
+          GetProgress(curr_bytes_diffed_ + curr_sig_bytes_read_ / kSigFactor,
+                      curr_filesize_ + server_filesize_ / kSigFactor);
+      OutputFileProgress(OutputType::kDiff, progress, finished);
+      break;
+    }
+
+    case State::kDelete:
+      // Should have been handled above.
+      assert(false);
+    case State::kIdle:
+      break;
+  }
+}
+
+void ProgressTracker::OutputFileProgress(OutputType type, double progress,
+                                         bool finished) const {
+  // Just in case some calculation wasn't 100% right.
+  if (finished) {
+    progress = 1.0;
+  }
+
+  double file_sec = file_timer_.ElapsedSeconds();
+  double file_eta_sec = file_sec / std::max(1e-3, progress) - file_sec;
+  double filespeed = 0;
+
+  // Don't bother trying to estimate transfer speed for signature.
+  if (type == OutputType::kCopy) {
+    filespeed = curr_filesize_ * progress / std::max(1e-3, file_sec);
+  } else if (type == OutputType::kDiff) {
+    // Take the sig time into account to estimate the effective transfer speed,
+    // using the actual diffing progress, not the combined diffing + signing
+    // progress.
+    double diff_progress = GetProgress(curr_bytes_diffed_, curr_filesize_);
+    double diff_sec = file_sec - sig_time_sec_;
+    double final_file_sec =
+        diff_sec / std::max(1e-3, diff_progress) + sig_time_sec_;
+    filespeed = curr_filesize_ / std::max(1e-3, final_file_sec);
+  }
+
+  if (json_) {
+    const char* op = type == OutputType::kCopy  ? "Copy"
+                     : type == OutputType::kSig ? "Sign"
+                                                : "Diff";
+
+    double total_progress, total_sec, total_eta_sec;
+    GetTotalProgressStats(&total_progress, &total_sec, &total_eta_sec);
+
+    Json::Value val;
+    val["file"] = curr_filepath_;
+    val["operation"] = op;
+    val["size"] = curr_filesize_;
+    val["bytes_per_second"] = filespeed;
+    val["duration"] = file_sec;
+    val["eta"] = file_eta_sec;
+    val["total_progress"] = total_progress;
+    val["total_duration"] = total_sec;
+    val["total_eta"] = total_eta_sec;
+    PrintJson(val, finished);
+  } else {
+    char ch = type == OutputType::kCopy  ? 'C'
+              : type == OutputType::kSig ? 'S'
+                                         : 'D';
+
+    int progress_percent = static_cast<int>(progress * 100);
+
+    uint64_t filesize = curr_filesize_;
+    const char* filesize_unit = FormatIntBytes(&filesize);
+    std::string filetime_str = FormatTime(file_sec);
+
+    const char* filespeed_unit = "B";
+    std::string filespeed_str = "---.-";
+
+    // Don't bother trying to estimate transfer speed for signature.
+    if (type != OutputType::kSig) {
+      filespeed_unit = FormatDoubleBytes(&filespeed);
+      filespeed_str = absl::StrFormat("%5.1f", filespeed);
+    }
+
+    const char* fileeta_str = "   ";
+    if (progress < 1.0) {
+      // While in progress, time is an ETA.
+      filetime_str = FormatTime(file_eta_sec);
+      fileeta_str = "ETA";
+    }
+
+    // file                S 50% 12345MB ---.-MB/s --:-- ETA  50% TOT 005:00 ETA
+    std::string txt = absl::StrFormat(
+        " %c%3i%% %5i%-2s "
+        "%s%-2s/s %s %3s "
+        "%s",
+        ch, progress_percent, filesize, filesize_unit, filespeed_str.c_str(),
+        filespeed_unit, filetime_str.c_str(), fileeta_str,
+        GetTotalProgressText().c_str());
+
+    int width = GetOutputWidth();
+    int file_width = std::max(12, width - static_cast<int>(txt.size()));
+    std::string short_path = ShortenFilePath(curr_filepath_, file_width);
+    PaddRight(&short_path, file_width);
+
+    printer_->Print(short_path + txt, finished, width);
+  }
+}
+
+void ProgressTracker::Print(std::string text, bool finished) const {
+  if (printer_->quiet()) return;
+
+  printer_->Print(std::move(text), finished, GetOutputWidth());
+}
+
+void ProgressTracker::PrintJson(const Json::Value& val, bool finished) const {
+  if (printer_->quiet()) return;
+
+  Json::FastWriter writer;
+  std::string json = writer.write(val);
+  if (!json.empty() && json.back() == '\n') json.pop_back();
+  printer_->Print(json, finished, 0);
+}
+
+double ProgressTracker::GetTotalProgress() const {
+  return GetProgress(total_bytes_copied_ + total_bytes_diffed_ +
+                         total_sig_bytes_read_ / kSigFactor,
+                     total_bytes_to_copy_ + total_bytes_to_diff_ +
+                         total_sig_bytes_ / kSigFactor);
+}
+
+void ProgressTracker::GetTotalProgressStats(double* total_progress,
+                                            double* total_sec,
+                                            double* total_eta_sec) const {
+  *total_progress = GetTotalProgress();
+  *total_sec = total_timer_.ElapsedSeconds();
+  *total_eta_sec = *total_sec / std::max(1e-3, *total_progress) - *total_sec;
+}
+
+std::string ProgressTracker::GetTotalProgressText() const {
+  double total_progress, total_sec, total_eta_sec;
+  GetTotalProgressStats(&total_progress, &total_sec, &total_eta_sec);
+  int total_progress_percent = static_cast<int>(total_progress * 100);
+
+  const char* total_eta_str = "   ";
+  if (total_progress < 1.0) {
+    // total_sec is an ETA.
+    total_sec = total_eta_sec;
+    total_eta_str = "ETA";
+  }
+  std::string total_sec_str = FormatTime(total_sec);
+
+  return absl::StrFormat("%3i%% TOT %s %s", total_progress_percent,
+                         total_sec_str.c_str(), total_eta_str);
+}
+
+int ProgressTracker::GetOutputWidth() const {
+  return fixed_output_width_ > 0 ? fixed_output_width_
+                                 : Util::GetConsoleWidth();
+}
+
+}  // namespace cdc_ft

cdc_rsync/progress_tracker.h

@@ -0,0 +1,244 @@
+/*
+ * Copyright 2022 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef CDC_RSYNC_PROGRESS_TRACKER_H_
+#define CDC_RSYNC_PROGRESS_TRACKER_H_
+
+#include <string>
+
+#include "cdc_rsync/base/cdc_interface.h"
+#include "cdc_rsync/file_finder_and_sender.h"
+#include "common/stopwatch.h"
+
+namespace Json {
+class Value;
+}
+
+namespace cdc_ft {
+
+class ProgressPrinter {
+ public:
+  ProgressPrinter(bool quiet, bool is_tty) : quiet_(quiet), is_tty_(is_tty) {}
+  virtual ~ProgressPrinter() = default;
+
+  virtual void Print(std::string text, bool newline, int output_width) = 0;
+
+  bool quiet() const { return quiet_; }
+  bool is_tty() const { return is_tty_; }
+
+ private:
+  const bool quiet_;
+  const bool is_tty_;
+};
+
+class ConsoleProgressPrinter : public ProgressPrinter {
+ public:
+  ConsoleProgressPrinter(bool quiet, bool is_tty);
+
+  // Prints |text| to stdout. Adds a line feed (\n) if |newline| is true or
+  // is_tty() is false (e.g. logging to a file). Otherwise, just adds a carriage
+  // return (\r), so that the next call to Output overwrites the current line.
+  // Fills the rest of the line up to |output_width| characters with spaces to
+  // properly overwrite the last line.
+  // No-op if quiet().
+  void Print(std::string text, bool newline, int output_width) override;
+};
+
+// Tracks progress of the various stages of rsync and displays them in a human-
+// readable manner.
+class ProgressTracker : public ReportCdcProgress,
+                        public ReportFindFilesProgress {
+ public:
+  // |verbosity| (number of -v arguments) impacts the display verbosity.
+  // 0 only prints total progress and ETA/time.
+  // 1 prints per-file process and ETA/time.
+  // |json| prints JSON progress.
+  // If |fixed_output_width| > 0, formats output to that width, otherwise, uses
+  // the console width.
+  ProgressTracker(ProgressPrinter* printer, int verbosity, bool json,
+                  int fixed_output_width = 0,
+                  SteadyClock* clock = DefaultSteadyClock::GetInstance());
+  ~ProgressTracker();
+
+  // Starts reporting finding all source files to copy.
+  // Must be in idle state. Must be called before ReportFileFound().
+  void StartFindFiles();
+
+  // Reports that a file has been found.
+  void ReportFileFound();
+
+  // Reports that a directory has been found.
+  void ReportDirFound();
+
+  // Prints out the 4 files numbers and stores the total bytes for progress
+  // calculations. See SendFileStatsResponse in messages.proto for more info.
+  void ReportFileStats(uint32_t num_missing_files,
+                       uint32_t num_extraneous_files,
+                       uint32_t num_matching_files, uint32_t num_changed_files,
+                       uint64_t total_missing_bytes,
+                       uint64_t total_changed_client_bytes,
+                       uint64_t total_changed_server_bytes,
+                       uint32_t num_missing_dirs, uint32_t num_extraneous_dirs,
+                       uint32_t num_matching_dirs, bool whole_file_arg = false,
+                       bool checksum_arg = false, bool delete_arg = false);
+
+  // Starts reporting the copy of the file at |filepath| of size |filesize|.
+  // Must be in idle state. Must be called before ReportCopyProgress().
+  void StartCopy(const std::string& filepath, uint64_t filesize);
+
+  // Reports that |num_bytes_copied| have been copied for the current file.
+  void ReportCopyProgress(uint64_t num_bytes_copied);
+
+  // Starts reporting the delta sync of the file at |filepath| of size
+  // |client_size|. |server_size| is the size of the corresponding file on the
+  // server.
+  // Must be in idle state. Must be called before ReportSyncProgress().
+  void StartSync(const std::string& filepath, uint64_t client_size,
+                 uint64_t server_size);
+
+  // ReportCdcProgress:
+
+  // Reports that |num_client_bytes_processed| of the current client-side file
+  // have been read and processed by the delta-transfer algorithm, and that
+  // |num_server_bytes_processed| of the current server-side file have been
+  // read and processed.
+  void ReportSyncProgress(uint64_t num_client_bytes_processed,
+                          uint64_t num_server_bytes_processed) override;
+
+  // Starts reporting deletion of extraneous files.
+  // Must be in idle state. Must be called before ReportFileDeleted().
+  void StartDeleteFiles();
+
+  // Reports that a file has been deleted.
+  void ReportFileDeleted(const std::string& filepath);
+
+  // Reports that a directory has been deleted.
+  void ReportDirDeleted(const std::string& filepath);
+
+  // Prints final stats (e.g. 100% progress for copy/diff), feeds line and
+  // resets state to idle. Must be called
+  // - when all files have been found,
+  // - after each file copy and
+  // - after each diff.
+  void Finish();
+
+  // Gets the time delay (in seconds) between two display updates. Returns a
+  // lower number (more updates) in TTY mode, e.g. when running from a terminal,
+  // and a higher number otherwise, e.g. when piping stdout to a file.
+  double GetDisplayDelaySecForTesting() const { return display_delay_sec_; }
+
+ private:
+  // Prints progress to the console. Rate-limited, so that it can be called
+  // often without performance overhead. |finished| should be set if the current
+  // item is finished. It adds a line feed.
+  // No-op if printer_->quiet().
+  void UpdateOutput(bool finished);
+
+  enum class OutputType { kCopy, kSig, kDiff };
+
+  // Prints out the progress of the current file (for copy/diff state).
+  // Used if |verbosity_| > 0 and not printer_->quiet().
+  void OutputFileProgress(OutputType type, double progress,
+                          bool finished) const;
+
+  // Wrapper for printer_->Print(), but checks printer_->quiet().
+  void Print(std::string text, bool finished) const;
+
+  // Prints the JSON value |val|.
+  void PrintJson(const Json::Value& val, bool finished) const;
+
+  // Returns the total progress (between 0 and 1).
+  double GetTotalProgress() const;
+
+  // Gets the |total_progress| (between 0 and 1), the total duration so far in
+  // |total_sec| and the estimated ETA (time left) in |total_eta_sec|.
+  void GetTotalProgressStats(double* total_progress, double* total_sec,
+                             double* total_eta_sec) const;
+
+  // Returns a string with the total progress, e.g. " 19% TOT 03:59 ETA".
+  std::string GetTotalProgressText() const;
+
+  // Returns |fixed_output_width_| if > 0 or Util::GetConsoleWidth().
+  int GetOutputWidth() const;
+
+  ProgressPrinter* const printer_;
+  const bool only_total_progress_ = false;
+  const bool json_ = false;
+  const int fixed_output_width_;
+  const double display_delay_sec_;
+
+  // Timer for total copy and diff time (not file search, that's not timed).
+  Stopwatch total_timer_;
+  // Timer for single file copy and diff time.
+  Stopwatch file_timer_;
+  // Timer for processing signatures of server files.
+  Stopwatch sig_timer_;
+  // Timer to limit the rate of console outputs.
+  Stopwatch print_timer_;
+
+  enum class State { kIdle, kSearch, kCopy, kSyncSig, kSyncDiff, kDelete };
+
+  State state_ = State::kIdle;
+
+  // Number of files found so far.
+  uint32_t files_found_ = 0;
+
+  // Number of directories found so far.
+  uint32_t dirs_found_ = 0;
+
+  // Path of the file currently copied or synced.
+  std::string curr_filepath_;
+  // Size of the file at |current_filepath_|.
+  uint64_t curr_filesize_ = 0;
+
+  // Number of bytes of the file at |current_filepath_| already copied.
+  uint64_t curr_bytes_copied_ = 0;
+  // Total number of bytes already copied.
+  uint64_t total_bytes_copied_ = 0;
+  // Total number of bytes to copy.
+  uint64_t total_bytes_to_copy_ = 0;
+
+  // Number of signature bytes read so far on the server.
+  uint64_t curr_sig_bytes_read_ = 0;
+  // Total number of signature bytes already processed on the server.
+  uint64_t total_sig_bytes_read_ = 0;
+  // Total number of signature bytes.
+  uint64_t total_sig_bytes_ = 0;
+  // Total size of the server-side file.
+  uint64_t server_filesize_ = 0;
+  // Duration for signature computation.
+  double sig_time_sec_ = 0.0;
+
+  // Number of bytes of the file at |current_filepath_| already diffed.
+  uint64_t curr_bytes_diffed_ = 0;
+  // Total number of bytes already diffed.
+  uint64_t total_bytes_diffed_ = 0;
+  // Total number of bytes to diff.
+  uint64_t total_bytes_to_diff_ = 0;
+
+  // Number of files deleted so far.
+  uint32_t files_deleted_ = 0;
+  // Total number of files to be deleted.
+  uint32_t total_files_to_delete_ = 0;
+  // Number of directories deleted so far.
+  uint32_t dirs_deleted_ = 0;
+  // Total number of directories to be deleted.
+  uint32_t total_dirs_to_delete_ = 0;
+};
+
+}  // namespace cdc_ft
+
+#endif  // CDC_RSYNC_PROGRESS_TRACKER_H_

cdc_rsync/progress_tracker_test.cc

@@ -0,0 +1,491 @@
+// Copyright 2022 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "cdc_rsync/progress_tracker.h"
+
+#include <ostream>
+
+#include "common/testing_clock.h"
+#include "gtest/gtest.h"
+
+namespace cdc_ft {
+namespace {
+
+// Create custom sizes, so that progress can be easily split up into steps, e.g.
+// kFileSize / 3 + kFileSize / 3,  + kFileSize / 3, and the end result is still
+// kFileSize.
+constexpr uint64_t kFileSize = 1 * 2 * 3 * 4 * 5 * 6;
+
+// Verbosity.
+const int kV0 = 0;
+const int kV1 = 1;
+
+const bool kJson = true;
+const bool kNoJson = false;
+
+const bool kQuiet = true;
+const bool kNoQuiet = false;
+
+const bool kTTY = true;
+const bool kNoTTY = false;
+
+// Class that just creates a list of things it was supposed to print.
+class FakeProgressPrinter : public ProgressPrinter {
+ public:
+  FakeProgressPrinter(bool quiet, bool is_tty)
+      : ProgressPrinter(quiet, is_tty) {}
+
+  void Print(std::string text, bool newline, int /*output_width*/) override {
+    lines_.push_back(text + (newline || !is_tty() ? "\n" : "\r"));
+  }
+
+  void ExpectLinesMatch(std::vector<std::string> expected_lines) {
+    EXPECT_EQ(lines_, expected_lines);
+  }
+
+ private:
+  std::vector<std::string> lines_;
+};
+
+class ProgressTrackerTest : public ::testing::Test {
+ protected:
+  // Returns the time (in ms) that needs to pass to trigger an output update.
+  double GetTriggerPrintTimeDeltaMs(const ProgressTracker& progress) {
+    return progress.GetDisplayDelaySecForTesting() * 1000 * 1.5;
+  }
+
+  TestingSteadyClock clock_;
+
+  uint32_t two_seconds_time_delta_ms_ = 2 * 1000;
+  uint32_t two_minutes_time_delta_ms_ = 2 * 60 * 1000;
+  uint32_t four_hours_time_delta_ms_ = 4 * 60 * 60 * 1000;
+  uint32_t eight_days_time_delta_ms_ = 8 * 24 * 60 * 60 * 1000;
+
+  const int output_width_ = 66;
+};
+
+TEST_F(ProgressTrackerTest, FindFiles) {
+  FakeProgressPrinter printer(kNoQuiet, kTTY);
+  ProgressTracker progress(&printer, kV0, kNoJson, output_width_, &clock_);
+
+  progress.StartFindFiles();
+  progress.ReportFileFound();
+  clock_.Advance(GetTriggerPrintTimeDeltaMs(progress));
+  progress.ReportFileFound();
+  progress.ReportFileFound();
+  clock_.Advance(GetTriggerPrintTimeDeltaMs(progress));
+  progress.ReportFileFound();
+  progress.ReportFileFound();
+  progress.Finish();
+
+  printer.ExpectLinesMatch({"2 file(s) and 0 folder(s) found\r",
+                            "4 file(s) and 0 folder(s) found\r",
+                            "5 file(s) and 0 folder(s) found\n"});
+}
+
+TEST_F(ProgressTrackerTest, FindFilesVerbose) {
+  FakeProgressPrinter printer(kNoQuiet, kTTY);
+  ProgressTracker progress(&printer, kV1, kNoJson, output_width_, &clock_);
+
+  progress.StartFindFiles();
+  clock_.Advance(GetTriggerPrintTimeDeltaMs(progress));
+  progress.ReportFileFound();
+  progress.Finish();
+
+  // Find files should be the same as non-verbose.
+  printer.ExpectLinesMatch({"1 file(s) and 0 folder(s) found\r",
+                            "1 file(s) and 0 folder(s) found\n"});
+}
+
+TEST_F(ProgressTrackerTest, CopyFiles) {
+  FakeProgressPrinter printer(kNoQuiet, kTTY);
+  ProgressTracker progress(&printer, kV0, kNoJson, output_width_, &clock_);
+
+  progress.StartCopy("file.txt", kFileSize);
+  progress.ReportCopyProgress(kFileSize / 3);
+  clock_.Advance(GetTriggerPrintTimeDeltaMs(progress));
+  progress.ReportCopyProgress(kFileSize / 3);
+  progress.ReportCopyProgress(kFileSize / 3);
+  progress.Finish();
+
+  printer.ExpectLinesMatch({"100% TOT 00:00    \r", "100% TOT 00:00    \n"});
+}
+
+TEST_F(ProgressTrackerTest, CopyFilesVerbose) {
+  FakeProgressPrinter printer(kNoQuiet, kTTY);
+  ProgressTracker progress(&printer, kV1, kNoJson, output_width_, &clock_);
+
+  progress.StartCopy("file.txt", kFileSize);
+  progress.ReportCopyProgress(kFileSize / 3);
+  clock_.Advance(GetTriggerPrintTimeDeltaMs(progress));
+  progress.ReportCopyProgress(kFileSize / 3);
+  progress.ReportCopyProgress(kFileSize / 3);
+  progress.Finish();
+
+  printer.ExpectLinesMatch(
+      {"file.txt      C 66%   720B    3.1KB/s 00:00 ETA 100% TOT 00:00    \r",
+       "file.txt      C100%   720B    4.7KB/s 00:00     100% TOT 00:00    \n"});
+}
+
+TEST_F(ProgressTrackerTest, SyncFiles) {
+  FakeProgressPrinter printer(kNoQuiet, kTTY);
+  ProgressTracker progress(&printer, kV0, kNoJson, output_width_, &clock_);
+
+  // 1 changed file.
+  progress.ReportFileStats(0, 0, 0, 1, 0, kFileSize, kFileSize, 0, 0, 0);
+
+  progress.StartSync("file.txt", kFileSize, kFileSize);
+  progress.ReportSyncProgress(0, kFileSize / 2);
+  clock_.Advance(GetTriggerPrintTimeDeltaMs(progress));
+  progress.ReportSyncProgress(0, kFileSize / 2);
+  clock_.Advance(GetTriggerPrintTimeDeltaMs(progress));
+  progress.ReportSyncProgress(kFileSize / 3, 0);
+  clock_.Advance(GetTriggerPrintTimeDeltaMs(progress));
+  progress.ReportSyncProgress(kFileSize / 3, 0);
+  progress.ReportSyncProgress(kFileSize / 3, 0);
+  progress.Finish();
+
+  printer.ExpectLinesMatch(
+      {"     0 file(s) and 0 folder(s) are not present on the instance and "
+       "will be copied.\n",
+       "     1 file(s) changed and will be updated.\n",
+       "     0 file(s) and 0 folder(s) match and do not have to be updated.\n",
+       "     0 file(s) and 0 folder(s) on the instance do not exist on this "
+       "machine.\n",
+       "  2% TOT 00:05 ETA\r", " 34% TOT 00:00 ETA\r", " 67% TOT 00:00 ETA\r",
+       "100% TOT 00:00    \n"});
+}
+
+TEST_F(ProgressTrackerTest, SyncFilesVerbose) {
+  FakeProgressPrinter printer(kNoQuiet, kTTY);
+  ProgressTracker progress(&printer, kV1, kNoJson, output_width_, &clock_);
+
+  // 1 changed file.
+  progress.ReportFileStats(0, 0, 0, 1, 0, kFileSize, kFileSize, 0, 0, 0);
+
+  progress.StartSync("file.txt", kFileSize, kFileSize);
+  clock_.Advance(two_seconds_time_delta_ms_);
+  progress.ReportSyncProgress(0, kFileSize / 3);
+  clock_.Advance(GetTriggerPrintTimeDeltaMs(progress));
+  progress.ReportSyncProgress(kFileSize / 3, 0);
+  clock_.Advance(GetTriggerPrintTimeDeltaMs(progress));
+  progress.ReportSyncProgress(0, kFileSize / 3);
+  clock_.Advance(GetTriggerPrintTimeDeltaMs(progress));
+  progress.ReportSyncProgress(0, kFileSize / 3);
+  progress.ReportSyncProgress(kFileSize / 3, 0);
+  clock_.Advance(GetTriggerPrintTimeDeltaMs(progress));
+  progress.ReportSyncProgress(kFileSize / 3, 0);
+  progress.Finish();
+
+  printer.ExpectLinesMatch(
+      {"     0 file(s) and 0 folder(s) are not present on the instance and "
+       "will be copied.\n",
+       "     1 file(s) changed and will be updated.\n",
+       "     0 file(s) and 0 folder(s) match and do not have to be updated.\n",
+       "     0 file(s) and 0 folder(s) on the instance do not exist on this "
+       "machine.\n",
+       "file.txt      S  0%   720B  ---.-B /s 04:03 ETA   0% TOT 04:03 ETA\r",
+       "file.txt      D 33%   720B  117.1B /s 00:04 ETA  33% TOT 00:04 ETA\r",
+       "file.txt      S 34%   720B  ---.-B /s 00:04 ETA  34% TOT 00:04 ETA\r",
+       "file.txt      S 34%   720B  ---.-B /s 00:04 ETA  34% TOT 00:04 ETA\r",
+       "file.txt      D100%   720B  276.9B /s 00:02     100% TOT 00:02    \r",
+       "file.txt      D100%   720B  276.9B /s 00:02     100% TOT 00:02    \n"});
+}
+
+TEST_F(ProgressTrackerTest, DeleteFiles) {
+  FakeProgressPrinter printer(kNoQuiet, kTTY);
+  ProgressTracker progress(&printer, kV0, kNoJson, output_width_, &clock_);
+
+  // 2 extraneous files.
+  progress.ReportFileStats(0, 4, 0, 0, 0, 0, 0, 0, 0, 0);
+  progress.StartDeleteFiles();
+  progress.ReportFileDeleted("file1.txt");
+  clock_.Advance(GetTriggerPrintTimeDeltaMs(progress));
+  progress.ReportFileDeleted("file2.txt");
+  progress.ReportFileDeleted("file3.txt");
+  progress.ReportFileDeleted("file4.txt");
+  progress.Finish();
+
+  printer.ExpectLinesMatch(
+      {"     0 file(s) and 0 folder(s) are not present on the instance and "
+       "will be copied.\n",
+       "     0 file(s) changed and will be updated.\n",
+       "     0 file(s) and 0 folder(s) match and do not have to be updated.\n",
+       "     4 file(s) and 0 folder(s) on the instance do not exist on this "
+       "machine.\n",
+       "2/4 file(s) and 0/0 folder(s) deleted.\r",
+       "4/4 file(s) and 0/0 folder(s) deleted.\n"});
+}
+
+TEST_F(ProgressTrackerTest, DeleteFilesVerbose) {
+  FakeProgressPrinter printer(kNoQuiet, kTTY);
+  ProgressTracker progress(&printer, kV1, kNoJson, output_width_, &clock_);
+
+  // 2 extraneous files.
+  progress.ReportFileStats(0, 2, 0, 0, 0, 0, 0, 0, 0, 0);
+  progress.StartDeleteFiles();
+  progress.ReportFileDeleted("file1.txt");
+  progress.ReportFileDeleted("file2.txt");
+  progress.Finish();
+
+  printer.ExpectLinesMatch(
+      {"     0 file(s) and 0 folder(s) are not present on the instance and "
+       "will be copied.\n",
+       "     0 file(s) changed and will be updated.\n",
+       "     0 file(s) and 0 folder(s) match and do not have to be updated.\n",
+       "     2 file(s) and 0 folder(s) on the instance do not exist on this "
+       "machine.\n",
+       "file1.txt                                            deleted 1 / 2\n",
+       "file2.txt                                            deleted 2 / 2\n"});
+}
+
+TEST_F(ProgressTrackerTest, DeleteFilesNoFiles) {
+  FakeProgressPrinter printer(kNoQuiet, kTTY);
+  ProgressTracker progress(&printer, kV0, kNoJson, output_width_, &clock_);
+
+  progress.ReportFileStats(0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
+  progress.StartDeleteFiles();
+  progress.Finish();
+
+  printer.ExpectLinesMatch(
+      {"     0 file(s) and 0 folder(s) are not present on the instance and "
+       "will be copied.\n",
+       "     0 file(s) changed and will be updated.\n",
+       "     0 file(s) and 0 folder(s) match and do not have to be updated.\n",
+       "     0 file(s) and 0 folder(s) on the instance do not exist on this "
+       "machine.\n"});
+}
+
+TEST_F(ProgressTrackerTest, SetFileStatsAndUnits) {
+  FakeProgressPrinter printer(kNoQuiet, kTTY);
+  ProgressTracker progress(&printer, kV1, kNoJson, output_width_, &clock_);
+
+  // Have a very large file and trigger different ETAs and transfer speeds.
+  constexpr uint64_t large_file_size = 2ull * 1024 * 1024 * 1024 * 1024;
+  progress.ReportFileStats(1, 0, 0, 0, large_file_size, 0, 0, 0, 0, 0);
+
+  progress.StartCopy("file.txt", large_file_size);
+  progress.ReportCopyProgress(large_file_size / 4);
+  clock_.Advance(GetTriggerPrintTimeDeltaMs(progress));
+  progress.ReportCopyProgress(large_file_size / 4);
+  clock_.Advance(two_seconds_time_delta_ms_);
+  progress.ReportCopyProgress(large_file_size / 4);
+  clock_.Advance(two_minutes_time_delta_ms_);
+  progress.ReportCopyProgress(large_file_size / 8);
+  clock_.Advance(four_hours_time_delta_ms_);
+  progress.ReportCopyProgress(large_file_size / 8);
+  clock_.Advance(eight_days_time_delta_ms_);
+  progress.Finish();
+
+  printer.ExpectLinesMatch(
+      {"     1 file(s) and 0 folder(s) are not present on the instance and "
+       "will be copied.\n",
+       "     0 file(s) changed and will be updated.\n",
+       "     0 file(s) and 0 folder(s) match and do not have to be updated.\n",
+       "     0 file(s) and 0 folder(s) on the instance do not exist on this "
+       "machine.\n",
+       "file.txt      C 50%  2048GB   6.7TB/s 00:00 ETA  50% TOT 00:00 ETA\r",
+       "file.txt      C 75%  2048GB 714.4GB/s 00:00 ETA  75% TOT 00:00 ETA\r",
+       "file.txt      C 87%  2048GB  14.7GB/s 00:17 ETA  87% TOT 00:17 ETA\r",
+       "file.txt     C100%  2048GB 144.4MB/s 04:02:02     100% TOT 04:02:02    "
+       "\r",
+       "file.txt     C100%  2048GB   3.0MB/s 196:02:02     100% TOT 196:02:02  "
+       "  \n"});
+}
+
+TEST_F(ProgressTrackerTest, QuietMode) {
+  FakeProgressPrinter printer(kQuiet, kTTY);
+  ProgressTracker progress(&printer, kV1, kNoJson, output_width_, &clock_);
+
+  progress.StartFindFiles();
+  progress.ReportFileFound();
+  progress.Finish();
+
+  // 1 missing, 1 extraneous, and 1 changed files
+  // 1 extraneous folder
+  progress.ReportFileStats(1, 1, 1, 0, kFileSize, kFileSize, kFileSize, 0, 0,
+                           1);
+
+  progress.StartCopy("file.txt", kFileSize);
+  progress.ReportCopyProgress(kFileSize);
+  progress.Finish();
+
+  progress.StartSync("file.txt", kFileSize, kFileSize);
+  progress.ReportSyncProgress(kFileSize, kFileSize);
+  progress.Finish();
+
+  progress.StartDeleteFiles();
+  progress.ReportFileDeleted("file.txt");
+  progress.ReportDirDeleted("folder");
+  progress.Finish();
+
+  printer.ExpectLinesMatch({});
+}
+
+TEST_F(ProgressTrackerTest, NoTTY) {
+  FakeProgressPrinter tty_printer(kNoQuiet, kTTY);
+  ProgressTracker tty_progress(&tty_printer, kV1, kNoJson, output_width_,
+                               &clock_);
+  double tty_delta_ms = GetTriggerPrintTimeDeltaMs(tty_progress);
+
+  // In no-TTY-mode (e.g. cdc_rsync .. > out.txt), the display rate should be
+  // lower (currently every 1 second instead of 0.1 seconds).
+  FakeProgressPrinter printer(kNoQuiet, kNoTTY);
+  ProgressTracker progress(&printer, kV1, kNoJson, output_width_, &clock_);
+  EXPECT_GT(GetTriggerPrintTimeDeltaMs(progress), tty_delta_ms);
+
+  progress.StartCopy("file.txt", kFileSize);
+  clock_.Advance(GetTriggerPrintTimeDeltaMs(progress));
+  progress.ReportCopyProgress(kFileSize / 3);
+  clock_.Advance(GetTriggerPrintTimeDeltaMs(progress));
+  progress.ReportCopyProgress(kFileSize / 3);
+  progress.ReportCopyProgress(kFileSize / 3);
+  progress.Finish();
+
+  printer.ExpectLinesMatch(
+      {"file.txt      C 33%   720B  160.0B /s 00:03 ETA 100% TOT 00:01    \n",
+       "file.txt      C 66%   720B  160.0B /s 00:01 ETA 100% TOT 00:03    \n",
+       "file.txt      C100%   720B  240.0B /s 00:03     100% TOT 00:03    \n"});
+}
+
+TEST_F(ProgressTrackerTest, JsonPerFile) {
+  FakeProgressPrinter printer(kNoQuiet, kNoTTY);
+  ProgressTracker progress(&printer, kV1, kJson, output_width_, &clock_);
+
+  progress.StartCopy("file.txt", kFileSize);
+  clock_.Advance(GetTriggerPrintTimeDeltaMs(progress));
+  progress.ReportCopyProgress(kFileSize / 3);
+  clock_.Advance(GetTriggerPrintTimeDeltaMs(progress));
+  progress.ReportCopyProgress(kFileSize / 3);
+  progress.ReportCopyProgress(kFileSize / 3);
+  progress.Finish();
+
+  printer.ExpectLinesMatch(
+      {"{\"bytes_per_second\":160.0,\"duration\":1.5,\"eta\":3.0,\"file\":"
+       "\"file.txt\",\"operation\":\"Copy\",\"size\":720,\"total_duration\":1."
+       "5,\"total_eta\":0.0,\"total_progress\":1.0}\n",
+       "{\"bytes_per_second\":160.0,\"duration\":3.0,\"eta\":1.5,\"file\":"
+       "\"file.txt\",\"operation\":\"Copy\",\"size\":720,\"total_duration\":3."
+       "0,\"total_eta\":0.0,\"total_progress\":1.0}\n",
+       "{\"bytes_per_second\":240.0,\"duration\":3.0,\"eta\":0.0,\"file\":"
+       "\"file.txt\",\"operation\":\"Copy\",\"size\":720,\"total_duration\":3."
+       "0,\"total_eta\":0.0,\"total_progress\":1.0}\n"});
+}
+
+TEST_F(ProgressTrackerTest, JsonTotal) {
+  FakeProgressPrinter printer(kNoQuiet, kNoTTY);
+  ProgressTracker progress(&printer, kV0, kJson, output_width_, &clock_);
+
+  progress.ReportFileStats(0, 0, 0, 1, 0, kFileSize, kFileSize, 0, 0, 0);
+
+  progress.StartCopy("file.txt", kFileSize);
+  clock_.Advance(GetTriggerPrintTimeDeltaMs(progress));
+  progress.ReportCopyProgress(kFileSize / 3);
+  clock_.Advance(GetTriggerPrintTimeDeltaMs(progress));
+  progress.ReportCopyProgress(kFileSize / 3);
+  progress.ReportCopyProgress(kFileSize / 3);
+  progress.Finish();
+
+  printer.ExpectLinesMatch(
+      {"     0 file(s) and 0 folder(s) are not present on the instance and "
+       "will be copied.\n",
+       "     1 file(s) changed and will be updated.\n",
+       "     0 file(s) and 0 folder(s) match and do not have to be updated.\n",
+       "     0 file(s) and 0 folder(s) on the instance do not exist on this "
+       "machine.\n",
+       "{\"total_duration\":1.5,\"total_eta\":3.1124999999999998,\"total_"
+       "progress\":0.32520325203252032}\n",
+       "{\"total_duration\":3.0,\"total_eta\":1.6124999999999998,\"total_"
+       "progress\":0.65040650406504064}\n"});
+}
+
+TEST_F(ProgressTrackerTest, SyncFilesWithWholeFile) {
+  FakeProgressPrinter printer(kNoQuiet, kTTY);
+  ProgressTracker progress(&printer, kV0, kNoJson, output_width_, &clock_);
+
+  // 1 changed file with -W arg.
+  progress.ReportFileStats(0, 0, 0, 1, 0, kFileSize, kFileSize, 0, 0, 0, true);
+  progress.StartCopy("file.txt", kFileSize);
+  progress.Finish();
+
+  printer.ExpectLinesMatch(
+      {"     0 file(s) and 0 folder(s) are not present on the instance and "
+       "will be copied.\n",
+       "     1 file(s) changed and will be copied due to -W/--whole-file.\n",
+       "     0 file(s) and 0 folder(s) match and do not have to be updated.\n",
+       "     0 file(s) and 0 folder(s) on the instance do not exist on this "
+       "machine.\n"});
+}
+
+TEST_F(ProgressTrackerTest, SyncFilesWithChecksum) {
+  FakeProgressPrinter printer(kNoQuiet, kTTY);
+  ProgressTracker progress(&printer, kV0, kNoJson, output_width_, &clock_);
+
+  // 1 matching file with -c arg.
+  progress.ReportFileStats(0, 0, 1, 0, 0, kFileSize, kFileSize, 0, 0, 0, false,
+                           true);
+  progress.StartCopy("file.txt", kFileSize);
+  progress.Finish();
+
+  printer.ExpectLinesMatch(
+      {"     0 file(s) and 0 folder(s) are not present on the instance and "
+       "will be copied.\n",
+       "     0 file(s) changed and will be updated.\n",
+       "     1 file(s) and 0 folder(s) have matching modified time and size, "
+       "but will be synced due to -c/--checksum.\n",
+       "     0 file(s) and 0 folder(s) on the instance do not exist on this "
+       "machine.\n"});
+}
+
+TEST_F(ProgressTrackerTest, SyncFilesWithChecksumAndWholeFile) {
+  FakeProgressPrinter printer(kNoQuiet, kTTY);
+  ProgressTracker progress(&printer, kV0, kNoJson, output_width_, &clock_);
+
+  // 1 changed file, 1 matching file. with -c and -W args.
+  progress.ReportFileStats(0, 0, 1, 1, 0, kFileSize, kFileSize, 0, 0, 0, true,
+                           true);
+  progress.StartCopy("file.txt", kFileSize);
+  progress.Finish();
+
+  printer.ExpectLinesMatch(
+      {"     0 file(s) and 0 folder(s) are not present on the instance and "
+       "will be copied.\n",
+       "     1 file(s) changed and will be copied due to -W/--whole-file.\n",
+       "     1 file(s) and 0 folder(s) have matching modified time and size, "
+       "but will be copied due to -c/--checksum and -W/--whole-file.\n",
+       "     0 file(s) and 0 folder(s) on the instance do not exist on this "
+       "machine.\n"});
+}
+
+TEST_F(ProgressTrackerTest, SyncFilesWithDelete) {
+  FakeProgressPrinter printer(kNoQuiet, kTTY);
+  ProgressTracker progress(&printer, kV0, kNoJson, output_width_, &clock_);
+
+  // 1 extraneous file with --delete arg.
+  progress.ReportFileStats(0, 1, 0, 0, 0, kFileSize, kFileSize, 0, 0, 0, false,
+                           false, true);
+  progress.StartCopy("file.txt", kFileSize);
+  progress.Finish();
+
+  printer.ExpectLinesMatch(
+      {"     0 file(s) and 0 folder(s) are not present on the instance and "
+       "will be copied.\n",
+       "     0 file(s) changed and will be updated.\n",
+       "     0 file(s) and 0 folder(s) match and do not have to be updated.\n",
+       "     1 file(s) and 0 folder(s) on the instance do not exist on this "
+       "machine and will be deleted due to --delete.\n"});
+}
+
+}  // namespace
+}  // namespace cdc_ft

cdc_rsync/protos/BUILD

@@ -0,0 +1,14 @@
+package(default_visibility = [
+    "//:__subpackages__",
+])
+
+proto_library(
+    name = "messages_proto",
+    srcs = ["messages.proto"],
+    visibility = ["//visibility:private"],
+)
+
+cc_proto_library(
+    name = "messages_cc_proto",
+    deps = [":messages_proto"],
+)

cdc_rsync/protos/messages.proto

@@ -0,0 +1,193 @@
+// Copyright 2022 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto3";
+option optimize_for = LITE_RUNTIME;
+
+package cdc_ft;
+
+// Used for testing.
+message TestRequest {
+  string message = 1;
+}
+
+// Notify server that subsequent messages are going to be compressed, e.g. when
+// the client is about to send missing files. Once all compressed data is sent,
+// the client waits for the ToggleCompressionResponse.
+message ToggleCompressionRequest {}
+
+// Notify client that all compressed messages have been received (e.g. all
+// missing files have been copied to the server) and that the client may switch
+// to uncompressed transfer again. This "write fence" or "sync point" is
+// necessary to prevent that the server reads past the compressed data because
+// it doesn't know where compressed data ends.
+message ToggleCompressionResponse {}
+
+// Send command line options to server.
+// The options largely match the command line args.
+message SetOptionsRequest {
+  message FilterRule {
+    enum Type {
+      TYPE_INCLUDE = 0;
+      TYPE_EXCLUDE = 1;
+    }
+
+    Type type = 1;
+    string pattern = 2;
+  }
+
+  string destination = 1;
+  bool delete = 2;
+  bool recursive = 3;
+  int32 verbosity = 4;
+  bool whole_file = 5;
+  bool compress = 6;
+  repeated FilterRule filter_rules = 7;
+  bool checksum = 8;
+  bool relative = 9;
+  bool dry_run = 10;
+  bool existing = 11;
+  string copy_dest = 12;
+}
+
+// Send file list to server.
+message AddFilesRequest {
+  message File {
+    string filename = 1;
+
+    // Linux epoch time. time_t, basically.
+    int64 modified_time = 2;
+
+    uint64 size = 3;
+  }
+
+  // Files are relative to this directory.
+  string directory = 1;
+
+  // Files in |directory|.
+  repeated File files = 2;
+
+  // Directories in |directory|.
+  repeated string dirs = 3;
+}
+
+// Send stats to client for logging purposes.
+message SendFileStatsResponse {
+  // Number of files present on the client, but not on the server.
+  uint32 num_missing_files = 1;
+
+  // Number of files present on the server, but not on the client.
+  uint32 num_extraneous_files = 2;
+
+  // Number of files present on both and matching.
+  uint32 num_matching_files = 3;
+
+  // Number of files present on both, but not matching.
+  uint32 num_changed_files = 4;
+
+  // Sum of the size of all missing files.
+  uint64 total_missing_bytes = 5;
+
+  // Sum of the client size of all changed files.
+  uint64 total_changed_client_bytes = 6;
+
+  // Sum of the server size of all changed files.
+  uint64 total_changed_server_bytes = 7;
+
+  // Number of directories present on the client, but not on the server.
+  uint32 num_missing_dirs = 8;
+
+  // Number of directories present on the server, but not on the client.
+  uint32 num_extraneous_dirs = 9;
+
+  // Number of directories present on both and matching.
+  uint32 num_matching_dirs = 10;
+}
+
+// Send indices of missing and changed files to client.
+message AddFileIndicesResponse {
+  // Client-side index of the file.
+  repeated uint32 client_indices = 1;
+}
+
+// Tell server that client will send data of a missing file.
+message SendMissingFileDataRequest {
+  // Server-side of the missing file.
+  uint32 server_index = 1;
+
+  // The actual file data is sent as raw data.
+}
+
+// Tell client that server is about to send signature data for diffing files.
+message SendSignatureResponse {
+  // Client-side index of the file.
+  uint32 client_index = 1;
+
+  // The total size of the server-side file.
+  uint64 server_file_size = 2;
+}
+
+// Send signatures for diffing file data to client. Uses SOA layout to save
+// bandwidth. The arrays are expected to be of the same length.
+message AddSignaturesResponse {
+  // Chunk sizes.
+  repeated uint32 sizes = 1;
+
+  // Chunk hashes, size should match (size of sizes) * (hash length).
+  bytes hashes = 2;
+}
+
+// Send patching information to server. Uses SOA layout to save bandwidth.
+// The arrays are expected to be of the same length.
+message AddPatchCommandsRequest {
+  enum Source {
+    // Use bytes [offset, offset + size) from |data| contained in this message.
+    // This means that no existing chunk can be reused.
+    SOURCE_DATA = 0;
+
+    // Use bytes [offset, offset + size) from the basis file.
+    // This means that an existing chunk can be reused.
+    SOURCE_BASIS_FILE = 1;
+  }
+
+  // Whether this is a reused chunk or a new chunk.
+  repeated Source sources = 1;
+
+  // Offsets into |data| or the basis file, depending on the source.
+  repeated uint64 offsets = 2;
+
+  // Sizes in |data| or the basis file, depending on the source.
+  repeated uint32 sizes = 3;
+
+  // Data bytes, for SOURCE_DATA.
+  bytes data = 4;
+}
+
+// Send list of to-be-deleted files to the client.
+message AddDeletedFilesResponse {
+  // Files are relative to this directory.
+  string directory = 1;
+
+  // Files in |directory|.
+  repeated string files = 2;
+
+  // Directories in |directory|.
+  repeated string dirs = 3;
+}
+
+// Tell server to shut the frick down.
+message ShutdownRequest {}
+
+// Ack for ShutdownRequest.
+message ShutdownResponse {}

cdc_rsync/testdata/file_finder_and_sender/a.txt

@@ -0,0 +1 @@
+1

cdc_rsync/testdata/file_finder_and_sender/b.txt

@@ -0,0 +1 @@
+22

cdc_rsync/testdata/file_finder_and_sender/c.txt

@@ -0,0 +1 @@
+333

cdc_rsync/testdata/file_finder_and_sender/subdir/d.txt

@@ -0,0 +1 @@
+4444

cdc_rsync/testdata/file_finder_and_sender/subdir/e.txt

@@ -0,0 +1 @@
+55555

cdc_rsync/testdata/parallel_file_opener/file1.txt

@@ -0,0 +1 @@
+data1

cdc_rsync/testdata/parallel_file_opener/file2.txt

@@ -0,0 +1 @@
+data2

cdc_rsync/testdata/parallel_file_opener/file3.txt

@@ -0,0 +1 @@
+data3

cdc_rsync/zstd_stream.cc

@@ -0,0 +1,182 @@
+// Copyright 2022 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "cdc_rsync/zstd_stream.h"
+
+#include <thread>
+
+#include "common/log.h"
+#include "common/status.h"
+#include "common/status_macros.h"
+
+namespace cdc_ft {
+namespace {
+
+// If the compressor gets less data than 1 buffer (128k) every 500 ms, then
+// trigger a flush. This happens when files with no changes are diff'ed (this
+// produces very low volume data). Flushing prevents that the server gets stale
+// and becomes overwhelmed later.
+constexpr absl::Duration kMinCompressPeriod = absl::Milliseconds(500);
+
+}  // namespace
+
+ZstdStream::ZstdStream(Socket* socket, int level, uint32_t num_threads)
+    : socket_(socket), cctx_(nullptr) {
+  status_ = WrapStatus(Initialize(level, num_threads),
+                       "Failed to initialize stream compressor");
+}
+
+ZstdStream::~ZstdStream() {
+  if (cctx_) {
+    ZSTD_freeCCtx(cctx_);
+    cctx_ = nullptr;
+  }
+
+  {
+    absl::MutexLock lock(&mutex_);
+    shutdown_ = true;
+  }
+  if (compressor_thread_.joinable()) {
+    compressor_thread_.join();
+  }
+}
+
+absl::Status ZstdStream::Write(const void* data, size_t size) {
+  absl::MutexLock lock(&mutex_);
+  if (!status_.ok()) return status_;
+
+  size_t data_bytes_left = size;
+  const char* data_ptr = static_cast<const char*>(data);
+  while (data_bytes_left > 0) {
+    // Wait until the compressor thread has consumed data from |in_buffer_|.
+    auto cond = [&]() {
+      return shutdown_ || in_buffer_.size() < in_buffer_.capacity() ||
+             !status_.ok();
+    };
+    mutex_.Await(absl::Condition(&cond));
+    if (shutdown_) return MakeStatus("Compression stream was shut down");
+    if (!status_.ok()) return status_;
+
+    // Copy data to input buffer.
+    size_t free_in_buffer_bytes = in_buffer_.capacity() - in_buffer_.size();
+    const size_t to_copy = std::min(data_bytes_left, free_in_buffer_bytes);
+    in_buffer_.append(data_ptr, to_copy);
+    data_bytes_left -= to_copy;
+    free_in_buffer_bytes -= to_copy;
+    data_ptr += to_copy;
+  }
+  return absl::OkStatus();
+}
+
+absl::Status ZstdStream::Flush() {
+  absl::MutexLock lock(&mutex_);
+  if (!status_.ok()) return status_;
+
+  last_chunk_ = true;
+  last_chunk_sent_ = false;
+
+  // Wait until data is flushed.
+  auto cond = [&]() { return shutdown_ || last_chunk_sent_ || !status_.ok(); };
+  mutex_.Await(absl::Condition(&cond));
+  if (shutdown_) return MakeStatus("Compression stream was shut down");
+  return status_;
+}
+
+absl::Status ZstdStream::Initialize(int level, uint32_t num_threads) {
+  cctx_ = ZSTD_createCCtx();
+  if (!cctx_) {
+    return MakeStatus("Failed to create compression context");
+  }
+
+  size_t res = ZSTD_CCtx_setParameter(cctx_, ZSTD_c_compressionLevel, level);
+  if (ZSTD_isError(res)) {
+    return MakeStatus("Failed to set compression level: %s",
+                      ZSTD_getErrorName(res));
+  }
+
+  // This fails if ZStd was not compiled with -DZSTD_MULTITHREAD.
+  res = ZSTD_CCtx_setParameter(cctx_, ZSTD_c_nbWorkers, num_threads);
+  if (ZSTD_isError(res)) {
+    return MakeStatus("Failed to set number of worker threads: %s",
+                      ZSTD_getErrorName(res));
+  }
+
+  {
+    absl::MutexLock lock(&mutex_);
+    in_buffer_.reserve(ZSTD_CStreamInSize());
+  }
+
+  compressor_thread_ = std::thread([this]() { ThreadCompressorMain(); });
+
+  return absl::OkStatus();
+}
+
+void ZstdStream::ThreadCompressorMain() {
+  std::vector<uint8_t> out_buffer;
+  out_buffer.resize(ZSTD_CStreamOutSize());
+
+  absl::MutexLock lock(&mutex_);
+  while (!shutdown_) {
+    // Wait for input data.
+    auto cond = [&]() {
+      return shutdown_ || last_chunk_ ||
+             in_buffer_.size() == in_buffer_.capacity();
+    };
+    bool flush =
+        !mutex_.AwaitWithTimeout(absl::Condition(&cond), kMinCompressPeriod);
+    if (shutdown_) {
+      return;
+    }
+
+    // If data arrives at a very slow rate (<1 buffer per kMinCompressPeriod),
+    // then flush the compression pipes.
+    const ZSTD_EndDirective mode = last_chunk_ ? ZSTD_e_end
+                                   : flush     ? ZSTD_e_flush
+                                               : ZSTD_e_continue;
+    LOG_DEBUG("Compressing %u bytes (mode=%s)", in_buffer_.size(),
+              mode == ZSTD_e_end     ? "end"
+              : mode == ZSTD_e_flush ? "flush"
+                                     : "continue");
+    ZSTD_inBuffer input = {in_buffer_.data(), in_buffer_.size(), 0};
+    bool finished = false;
+    do {
+      ZSTD_outBuffer output = {out_buffer.data(), out_buffer.size(), 0};
+      size_t remaining = ZSTD_compressStream2(cctx_, &output, &input, mode);
+      if (ZSTD_isError(remaining)) {
+        status_ = MakeStatus("Failed to compress data: %s",
+                             ZSTD_getErrorName(remaining));
+        return;
+      }
+
+      if (output.pos > 0) {
+        status_ = socket_->Send(output.dst, output.pos);
+        if (!status_.ok()) return;
+      }
+
+      finished = mode != ZSTD_e_continue ? (remaining == 0)
+                                         : (input.pos == input.size);
+    } while (!finished);
+
+    if (last_chunk_) {
+      last_chunk_ = false;
+      last_chunk_sent_ = true;
+    }
+
+    // zstd should only return 0 when the input is consumed.
+    assert(input.pos == input.size);
+    in_buffer_.clear();
+  }
+}
+
+}  // namespace cdc_ft

cdc_rsync/zstd_stream.h

@@ -0,0 +1,65 @@
+/*
+ * Copyright 2022 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef CDC_RSYNC_ZSTD_STREAM_H_
+#define CDC_RSYNC_ZSTD_STREAM_H_
+
+#include <thread>
+
+#include "absl/status/status.h"
+#include "absl/synchronization/mutex.h"
+#include "cdc_rsync/base/socket.h"
+#include "common/buffer.h"
+#include "lib/zstd.h"
+
+namespace cdc_ft {
+
+// Streaming compression using zstd.
+class ZstdStream {
+ public:
+  ZstdStream(Socket* socket, int level, uint32_t num_threads);
+  ~ZstdStream();
+
+  // Sends the given |data| to the compressor.
+  absl::Status Write(const void* data, size_t size) ABSL_LOCKS_EXCLUDED(mutex_);
+
+  // Flushes all remaining data and sends the compressed data to the socket.
+  absl::Status Flush() ABSL_LOCKS_EXCLUDED(mutex_);
+
+ private:
+  // Initializes the compressor and related data.
+  absl::Status Initialize(int level, uint32_t num_threads)
+      ABSL_LOCKS_EXCLUDED(mutex_);
+
+  // Compressor thread, pushes |in_buffer_| to the zstd compressor and sends
+  // compressed data to the socket.
+  void ThreadCompressorMain() ABSL_LOCKS_EXCLUDED(mutex_);
+
+  Socket* const socket_;
+  ZSTD_CCtx* cctx_;
+
+  absl::Mutex mutex_;
+  Buffer in_buffer_ ABSL_GUARDED_BY(mutex_);
+  bool shutdown_ ABSL_GUARDED_BY(mutex_) = false;
+  bool last_chunk_ ABSL_GUARDED_BY(mutex_) = false;
+  bool last_chunk_sent_ ABSL_GUARDED_BY(mutex_) = false;
+  absl::Status status_ ABSL_GUARDED_BY(mutex_);
+  std::thread compressor_thread_;
+};
+
+}  // namespace cdc_ft
+
+#endif  // CDC_RSYNC_ZSTD_STREAM_H_

cdc_rsync/zstd_stream_test.cc

@@ -0,0 +1,72 @@
+// Copyright 2022 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "cdc_rsync/zstd_stream.h"
+
+#include "cdc_rsync/base/fake_socket.h"
+#include "cdc_rsync_server/unzstd_stream.h"
+#include "common/status_test_macros.h"
+#include "gtest/gtest.h"
+
+namespace cdc_ft {
+namespace {
+
+class ZstdStreamTest : public ::testing::Test {
+ protected:
+  FakeSocket socket_;
+  ZstdStream cstream_{&socket_, /*level=*/6, /*num_threads=*/8};
+  UnzstdStream dstream_{&socket_};
+};
+
+TEST_F(ZstdStreamTest, Small) {
+  const std::string want = "Lorem ipsum gibberisulum foobarberis";
+  EXPECT_OK(cstream_.Write(want.data(), want.size()));
+  EXPECT_OK(cstream_.Flush());
+
+  Buffer buff(1024);
+  size_t bytes_read;
+  bool eof = false;
+  EXPECT_OK(dstream_.Read(buff.data(), buff.size(), &bytes_read, &eof));
+  EXPECT_TRUE(eof);
+  std::string got(buff.data(), bytes_read);
+  EXPECT_EQ(got, want);
+}
+
+TEST_F(ZstdStreamTest, Large) {
+  Buffer want(1024 * 1024 * 10 + 12345);
+  constexpr uint64_t prime = 919393;
+  for (size_t n = 0; n < want.size(); ++n) {
+    want.data()[n] = ((n * prime) % 26) + 'a';
+  }
+
+  constexpr int kChunkSize = 19 * 1024;
+  for (size_t pos = 0; pos < want.size(); pos += kChunkSize) {
+    size_t size = std::min<size_t>(kChunkSize, want.size() - pos);
+    EXPECT_OK(cstream_.Write(want.data() + pos, size));
+  }
+  EXPECT_OK(cstream_.Flush());
+
+  bool eof = false;
+  Buffer buff(128 * 1024);
+  Buffer got;
+  while (!eof) {
+    size_t bytes_read;
+    EXPECT_OK(dstream_.Read(buff.data(), buff.size(), &bytes_read, &eof));
+    got.append(buff.data(), bytes_read);
+  }
+  EXPECT_EQ(want, got);
+}
+
+}  // namespace
+}  // namespace cdc_ft