Releasing the former Stadia file transfer tools

The tools allow efficient and fast synchronization of large directory
trees from a Windows workstation to a Linux target machine.

cdc_rsync* support efficient copy of files by using content-defined
chunking (CDC) to identify chunks within files that can be reused.

asset_stream_manager + cdc_fuse_fs support efficient streaming of a
local directory to a remote virtual file system based on FUSE. It also
employs CDC to identify and reuse unchanged data chunks.

asset_stream_manager/.gitignore

@@ -0,0 +1,3 @@
+x64/*
+*.log
+*.user

asset_stream_manager/BUILD

@@ -0,0 +1,186 @@
+package(default_visibility = [
+    "//:__subpackages__",
+])
+
+cc_binary(
+    name = "asset_stream_manager",
+    srcs = ["main.cc"],
+    data = [":roots_pem"],
+    deps = [
+        ":asset_stream_config",
+        ":session_management_server",
+        "//common:log",
+        "//common:path",
+        "//common:sdk_util",
+        "//data_store:data_provider",
+    ],
+)
+
+cc_library(
+    name = "asset_stream_server",
+    srcs = [
+        "asset_stream_server.cc",
+        "grpc_asset_stream_server.cc",
+        "testing_asset_stream_server.cc",
+    ],
+    hdrs = [
+        "asset_stream_server.h",
+        "grpc_asset_stream_server.h",
+        "testing_asset_stream_server.h",
+    ],
+    deps = [
+        "//common:grpc_status",
+        "//common:log",
+        "//common:path",
+        "//common:status",
+        "//common:status_macros",
+        "//common:thread_safe_map",
+        "//data_store",
+        "//manifest:manifest_updater",
+        "//proto:asset_stream_service_grpc_proto",
+        "@com_google_absl//absl/strings:str_format",
+        "@com_google_absl//absl/time",
+    ],
+)
+
+cc_library(
+    name = "asset_stream_config",
+    srcs = ["asset_stream_config.cc"],
+    hdrs = ["asset_stream_config.h"],
+    deps = [
+        ":multi_session",
+        "//absl_helper:jedec_size_flag",
+        "//common:log",
+        "//common:path",
+        "//common:status_macros",
+        "@com_github_jsoncpp//:jsoncpp",
+        "@com_google_absl//absl/flags:parse",
+    ],
+)
+
+cc_library(
+    name = "cdc_fuse_manager",
+    srcs = ["cdc_fuse_manager.cc"],
+    hdrs = ["cdc_fuse_manager.h"],
+    deps = [
+        "//cdc_fuse_fs:constants",
+        "//common:gamelet_component",
+        "//common:remote_util",
+        "//common:status_macros",
+        "@com_google_absl//absl/status",
+        "@com_google_absl//absl/strings:str_format",
+    ],
+)
+
+cc_library(
+    name = "session_management_server",
+    srcs = [
+        "background_service_impl.cc",
+        "background_service_impl.h",
+        "local_assets_stream_manager_service_impl.cc",
+        "local_assets_stream_manager_service_impl.h",
+        "session_management_server.cc",
+        "session_manager.cc",
+        "session_manager.h",
+    ],
+    hdrs = ["session_management_server.h"],
+    deps = [
+        ":multi_session",
+        "//common:grpc_status",
+        "//common:log",
+        "//common:status_macros",
+        "//common:util",
+        "//manifest:manifest_updater",
+        "//metrics",
+        "//proto:background_service_grpc_proto",
+        "//proto:local_assets_stream_manager_grpc_proto",
+        "@com_google_absl//absl/strings",
+    ],
+)
+
+cc_library(
+    name = "multi_session",
+    srcs = [
+        "multi_session.cc",
+        "session.cc",
+        "session.h",
+    ],
+    hdrs = [
+        "multi_session.h",
+        "session_config.h",
+    ],
+    deps = [
+        ":asset_stream_server",
+        ":cdc_fuse_manager",
+        ":metrics_recorder",
+        "//common:file_watcher",
+        "//common:log",
+        "//common:path",
+        "//common:port_manager",
+        "//common:process",
+        "//common:remote_util",
+        "//common:sdk_util",
+        "//common:status_macros",
+        "//common:stopwatch",
+        "//data_store:disk_data_store",
+        "//manifest:manifest_printer",
+        "//manifest:manifest_updater",
+        "@com_google_absl//absl/status",
+    ],
+)
+
+cc_test(
+    name = "multi_session_test",
+    srcs = ["multi_session_test.cc"],
+    data = [":all_test_data"],
+    deps = [
+        ":multi_session",
+        "//common:test_main",
+        "//manifest:manifest_test_base",
+        "@com_google_googletest//:gtest",
+    ],
+)
+
+cc_library(
+    name = "metrics_recorder",
+    srcs = ["metrics_recorder.cc"],
+    hdrs = ["metrics_recorder.h"],
+    deps = [
+        "//common:log",
+        "//common:util",
+        "//metrics",
+        "//metrics:enums",
+        "//metrics:messages",
+        "@com_google_absl//absl/status",
+    ],
+)
+
+cc_test(
+    name = "metrics_recorder_test",
+    srcs = ["metrics_recorder_test.cc"],
+    deps = [
+        ":metrics_recorder",
+        "//common:status_test_macros",
+        "//common:test_main",
+        "//metrics",
+        "@com_google_googletest//:gtest",
+    ],
+)
+
+# Copy roots.pem to the output folder, required for authenticated gRPC.
+genrule(
+    name = "roots_pem",
+    srcs = ["@com_github_grpc_grpc//:root_certificates"],
+    outs = ["roots.pem"],
+    cmd = "cp $(location @com_github_grpc_grpc//:root_certificates) $(location roots.pem)",
+)
+
+filegroup(
+    name = "all_test_sources",
+    srcs = glob(["*_test.cc"]),
+)
+
+filegroup(
+    name = "all_test_data",
+    srcs = glob(["testdata/**"]),
+)

asset_stream_manager/asset_stream_config.cc

@@ -0,0 +1,184 @@
+// Copyright 2022 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "asset_stream_manager/asset_stream_config.h"
+
+#include <sstream>
+
+#include "absl/flags/flag.h"
+#include "absl/flags/parse.h"
+#include "absl/strings/str_format.h"
+#include "absl/strings/str_join.h"
+#include "absl_helper/jedec_size_flag.h"
+#include "common/buffer.h"
+#include "common/path.h"
+#include "common/status_macros.h"
+#include "json/json.h"
+
+ABSL_DECLARE_FLAG(std::string, src_dir);
+ABSL_DECLARE_FLAG(std::string, instance_ip);
+ABSL_DECLARE_FLAG(uint16_t, instance_port);
+ABSL_DECLARE_FLAG(int, verbosity);
+ABSL_DECLARE_FLAG(bool, debug);
+ABSL_DECLARE_FLAG(bool, singlethreaded);
+ABSL_DECLARE_FLAG(bool, stats);
+ABSL_DECLARE_FLAG(bool, quiet);
+ABSL_DECLARE_FLAG(bool, check);
+ABSL_DECLARE_FLAG(bool, log_to_stdout);
+ABSL_DECLARE_FLAG(cdc_ft::JedecSize, cache_capacity);
+ABSL_DECLARE_FLAG(uint32_t, cleanup_timeout);
+ABSL_DECLARE_FLAG(uint32_t, access_idle_timeout);
+ABSL_DECLARE_FLAG(int, manifest_updater_threads);
+ABSL_DECLARE_FLAG(int, file_change_wait_duration_ms);
+
+// Declare AS20 flags, so that AS30 can be used on older SDKs simply by
+// replacing the binary. Note that the RETIRED_FLAGS macro can't be used
+// because the flags contain dashes. This code mimics the macro.
+absl::flags_internal::RetiredFlag<std::string> RETIRED_FLAGS_session_ports;
+absl::flags_internal::RetiredFlag<std::string> RETIRED_FLAGS_gm_mount_point;
+absl::flags_internal::RetiredFlag<bool> RETIRED_FLAGS_allow_edge;
+
+const auto RETIRED_FLAGS_REG_session_ports =
+    (RETIRED_FLAGS_session_ports.Retire("session-ports"),
+     ::absl::flags_internal::FlagRegistrarEmpty{});
+const auto RETIRED_FLAGS_REG_gm_mount_point =
+    (RETIRED_FLAGS_gm_mount_point.Retire("gamelet-mount-point"),
+     ::absl::flags_internal::FlagRegistrarEmpty{});
+const auto RETIRED_FLAGS_REG_allow_edge =
+    (RETIRED_FLAGS_allow_edge.Retire("allow-edge"),
+     ::absl::flags_internal::FlagRegistrarEmpty{});
+
+namespace cdc_ft {
+
+AssetStreamConfig::AssetStreamConfig() {
+  src_dir_ = absl::GetFlag(FLAGS_src_dir);
+  instance_ip_ = absl::GetFlag(FLAGS_instance_ip);
+  instance_port_ = absl::GetFlag(FLAGS_instance_port);
+  session_cfg_.verbosity = absl::GetFlag(FLAGS_verbosity);
+  session_cfg_.fuse_debug = absl::GetFlag(FLAGS_debug);
+  session_cfg_.fuse_singlethreaded = absl::GetFlag(FLAGS_singlethreaded);
+  session_cfg_.stats = absl::GetFlag(FLAGS_stats);
+  session_cfg_.quiet = absl::GetFlag(FLAGS_quiet);
+  session_cfg_.fuse_check = absl::GetFlag(FLAGS_check);
+  log_to_stdout_ = absl::GetFlag(FLAGS_log_to_stdout);
+  session_cfg_.fuse_cache_capacity = absl::GetFlag(FLAGS_cache_capacity).Size();
+  session_cfg_.fuse_cleanup_timeout_sec = absl::GetFlag(FLAGS_cleanup_timeout);
+  session_cfg_.fuse_access_idle_timeout_sec =
+      absl::GetFlag(FLAGS_access_idle_timeout);
+  session_cfg_.manifest_updater_threads =
+      absl::GetFlag(FLAGS_manifest_updater_threads);
+  session_cfg_.file_change_wait_duration_ms =
+      absl::GetFlag(FLAGS_file_change_wait_duration_ms);
+}
+
+AssetStreamConfig::~AssetStreamConfig() = default;
+
+absl::Status AssetStreamConfig::LoadFromFile(const std::string& path) {
+  Buffer buffer;
+  RETURN_IF_ERROR(path::ReadFile(path, &buffer));
+
+  Json::Value config;
+  Json::Reader reader;
+  if (!reader.parse(buffer.data(), buffer.data() + buffer.size(), config,
+                    false)) {
+    return absl::InvalidArgumentError(
+        absl::StrFormat("Failed to parse config file '%s': %s", path,
+                        reader.getFormattedErrorMessages()));
+  }
+
+#define ASSIGN_VAR(var, flag, type)        \
+  do {                                     \
+    if (config.isMember(#flag)) {          \
+      var = config[#flag].as##type();      \
+      flags_read_from_file_.insert(#flag); \
+    }                                      \
+  } while (0)
+
+  ASSIGN_VAR(src_dir_, src_dir, String);
+  ASSIGN_VAR(session_cfg_.verbosity, verbosity, Int);
+  ASSIGN_VAR(session_cfg_.fuse_debug, debug, Bool);
+  ASSIGN_VAR(session_cfg_.fuse_singlethreaded, singlethreaded, Bool);
+  ASSIGN_VAR(session_cfg_.stats, stats, Bool);
+  ASSIGN_VAR(session_cfg_.quiet, quiet, Bool);
+  ASSIGN_VAR(session_cfg_.fuse_check, check, Bool);
+  ASSIGN_VAR(log_to_stdout_, log_to_stdout, Bool);
+  ASSIGN_VAR(session_cfg_.fuse_cleanup_timeout_sec, cleanup_timeout, Int);
+  ASSIGN_VAR(session_cfg_.fuse_access_idle_timeout_sec, access_idle_timeout,
+             Int);
+  ASSIGN_VAR(session_cfg_.manifest_updater_threads, manifest_updater_threads,
+             Int);
+  ASSIGN_VAR(session_cfg_.file_change_wait_duration_ms,
+             file_change_wait_duration_ms, Int);
+
+  // cache_capacity requires Jedec size conversion.
+  constexpr char kCacheCapacity[] = "cache_capacity";
+  if (config.isMember(kCacheCapacity)) {
+    JedecSize cache_capacity;
+    std::string error;
+    if (AbslParseFlag(config[kCacheCapacity].asString(), &cache_capacity,
+                      &error)) {
+      session_cfg_.fuse_cache_capacity = cache_capacity.Size();
+      flags_read_from_file_.insert(kCacheCapacity);
+    } else {
+      // Note that |error| can't be logged here since this code runs before
+      // logging is initialized.
+      flag_read_errors_[kCacheCapacity] = error;
+    }
+  }
+
+#undef ASSIGN_VAR
+
+  return absl::OkStatus();
+}  // namespace cdc_ft
+
+std::string AssetStreamConfig::ToString() {
+  std::ostringstream ss;
+  ss << "src_dir                      = " << src_dir_ << std::endl;
+  ss << "verbosity                    = " << session_cfg_.verbosity
+     << std::endl;
+  ss << "debug                        = " << session_cfg_.fuse_debug
+     << std::endl;
+  ss << "singlethreaded               = " << session_cfg_.fuse_singlethreaded
+     << std::endl;
+  ss << "stats                        = " << session_cfg_.stats << std::endl;
+  ss << "quiet                        = " << session_cfg_.quiet << std::endl;
+  ss << "check                        = " << session_cfg_.fuse_check
+     << std::endl;
+  ss << "log_to_stdout                = " << log_to_stdout_ << std::endl;
+  ss << "cache_capacity               = " << session_cfg_.fuse_cache_capacity
+     << std::endl;
+  ss << "cleanup_timeout              = "
+     << session_cfg_.fuse_cleanup_timeout_sec << std::endl;
+  ss << "access_idle_timeout          = "
+     << session_cfg_.fuse_access_idle_timeout_sec << std::endl;
+  ss << "manifest_updater_threads     = "
+     << session_cfg_.manifest_updater_threads << std::endl;
+  ss << "file_change_wait_duration_ms = "
+     << session_cfg_.file_change_wait_duration_ms << std::endl;
+  return ss.str();
+}
+
+std::string AssetStreamConfig::GetFlagsReadFromFile() {
+  return absl::StrJoin(flags_read_from_file_, ", ");
+}
+
+std::string AssetStreamConfig::GetFlagReadErrors() {
+  std::string error_str;
+  for (const auto& [flag, error] : flag_read_errors_)
+    error_str += absl::StrFormat("%sFailed to read '%s': %s",
+                                 error_str.empty() ? "" : "\n", flag, error);
+  return error_str;
+}
+
+}  // namespace cdc_ft

asset_stream_manager/asset_stream_config.h

@@ -0,0 +1,107 @@
+/*
+ * Copyright 2022 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef ASSET_STREAM_MANAGER_ASSET_STREAM_CONFIG_H_
+#define ASSET_STREAM_MANAGER_ASSET_STREAM_CONFIG_H_
+
+#include <map>
+#include <set>
+#include <string>
+
+#include "absl/status/status.h"
+#include "asset_stream_manager/session_config.h"
+
+namespace cdc_ft {
+
+// Class containing all configuration settings for asset streaming.
+// Reads flags from the command line and optionally applies overrides from
+// a json file.
+class AssetStreamConfig {
+ public:
+  // Constructs the configuration by applying command line flags.
+  AssetStreamConfig();
+  ~AssetStreamConfig();
+
+  // Loads a configuration from the JSON file at |path| and overrides any config
+  // values that are set in this file. Sample json file:
+  // {
+  //   "src_dir":"C:\\path\\to\\assets",
+  //   "verbosity":3,
+  //   "debug":0,
+  //   "singlethreaded":0,
+  //   "stats":0,
+  //   "quiet":0,
+  //   "check":0,
+  //   "log_to_stdout":0,
+  //   "cache_capacity":"150G",
+  //   "cleanup_timeout":300,
+  //   "access_idle_timeout":5,
+  //   "manifest_updater_threads":4,
+  //   "file_change_wait_duration_ms":500
+  // }
+  // Returns NotFoundError if the file does not exist.
+  // Returns InvalidArgumentError if the file is not valid JSON.
+  absl::Status LoadFromFile(const std::string& path);
+
+  // Returns a string with all config values, suitable for logging.
+  std::string ToString();
+
+  // Gets a comma-separated list of flags that were read from the JSON file.
+  // These flags override command line flags.
+  std::string GetFlagsReadFromFile();
+
+  // Gets a newline-separated list of errors for each flag that could not be
+  // read from the JSON file.
+  std::string GetFlagReadErrors();
+
+  // Workstation directory to stream. Should usually be empty since mounts are
+  // triggered by the CLI or the partner portal via a gRPC call, but useful
+  // during development.
+  const std::string& src_dir() const { return src_dir_; }
+
+  // IP address of the instance to stream to. Should usually be empty since
+  // mounts are triggered by the CLI or the partner portal via a gRPC call, but
+  // useful during development.
+  const std::string& instance_ip() const { return instance_ip_; }
+
+  // IP address of the instance to stream to. Should usually be unset (0) since
+  // mounts are triggered by the CLI or the partner portal via a gRPC call, but
+  // useful during development.
+  const uint16_t instance_port() const { return instance_port_; }
+
+  // Session configuration.
+  const SessionConfig session_cfg() const { return session_cfg_; }
+
+  // Whether to log to a file or to stdout.
+  bool log_to_stdout() const { return log_to_stdout_; }
+
+ private:
+  std::string src_dir_;
+  std::string instance_ip_;
+  uint16_t instance_port_ = 0;
+  SessionConfig session_cfg_;
+  bool log_to_stdout_ = false;
+
+  // Use a set, so the flags are sorted alphabetically.
+  std::set<std::string> flags_read_from_file_;
+
+  // Maps flags to errors occurred while reading this flag.
+  std::map<std::string, std::string> flag_read_errors_;
+};
+
+};  // namespace cdc_ft
+
+#endif  // ASSET_STREAM_MANAGER_ASSET_STREAM_CONFIG_H_

asset_stream_manager/asset_stream_manager.vcxproj

@@ -0,0 +1,90 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project DefaultTargets="Build" ToolsVersion="15.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <ItemGroup Label="ProjectConfigurations">
+    <ProjectConfiguration Include="Debug|x64">
+      <Configuration>Debug</Configuration>
+      <Platform>x64</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Release|x64">
+      <Configuration>Release</Configuration>
+      <Platform>x64</Platform>
+    </ProjectConfiguration>
+  </ItemGroup>
+  <PropertyGroup Label="Globals">
+    <VCProjectVersion>15.0</VCProjectVersion>
+    <ProjectGuid>{84D81562-D66C-4A60-9F48-2696D7D81D26}</ProjectGuid>
+    <Keyword>Win32Proj</Keyword>
+    <RootNamespace>cdc_rsync</RootNamespace>
+    <WindowsTargetPlatformVersion Condition="$(VisualStudioVersion) == 15">$([Microsoft.Build.Utilities.ToolLocationHelper]::GetLatestSDKTargetPlatformVersion('Windows', '10.0'))</WindowsTargetPlatformVersion>
+    <WindowsTargetPlatformVersion Condition="$(VisualStudioVersion) == 16">10.0</WindowsTargetPlatformVersion>
+  </PropertyGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
+    <ConfigurationType>Makefile</ConfigurationType>
+    <UseDebugLibraries>true</UseDebugLibraries>
+    <PlatformToolset Condition="$(VisualStudioVersion) == 15">v141</PlatformToolset>
+    <PlatformToolset Condition="$(VisualStudioVersion) == 16">v142</PlatformToolset>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
+    <ConfigurationType>Makefile</ConfigurationType>
+    <UseDebugLibraries>false</UseDebugLibraries>
+    <PlatformToolset Condition="$(VisualStudioVersion) == 15">v141</PlatformToolset>
+    <PlatformToolset Condition="$(VisualStudioVersion) == 16">v142</PlatformToolset>
+  </PropertyGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
+  <ImportGroup Label="Shared">
+    <Import Project="..\all_files.vcxitems" Label="Shared" />
+  </ImportGroup>
+  <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
+    <OutDir>$(SolutionDir)bazel-out\x64_windows-dbg\bin\asset_stream_manager\</OutDir>
+    <NMakePreprocessorDefinitions>UNICODE</NMakePreprocessorDefinitions>
+    <AdditionalOptions>/std:c++17</AdditionalOptions>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
+    <OutDir>$(SolutionDir)bazel-out\x64_windows-opt\bin\asset_stream_manager\</OutDir>
+    <NMakePreprocessorDefinitions>UNICODE</NMakePreprocessorDefinitions>
+    <AdditionalOptions>/std:c++17</AdditionalOptions>
+  </PropertyGroup>
+  <ItemGroup>
+    <ProjectReference Include="..\cdc_fuse_fs\cdc_fuse_fs.vcxproj">
+      <Project>{a537310c-0571-43d5-b7fe-c867f702294f}</Project>
+      <ReferenceOutputAssembly>false</ReferenceOutputAssembly>
+      <LinkLibraryDependencies>false</LinkLibraryDependencies>
+    </ProjectReference>
+  </ItemGroup>
+  <!-- Prevent console from being closed -->
+  <ItemDefinitionGroup>
+    <Link>
+      <SubSystem>Console</SubSystem>
+    </Link>
+  </ItemDefinitionGroup>
+  <!-- Bazel setup -->
+  <PropertyGroup>
+    <BazelTargets>//asset_stream_manager</BazelTargets>
+    <BazelOutputFile>asset_stream_manager.exe</BazelOutputFile>
+    <BazelIncludePaths>..\;..\third_party\absl;..\third_party\jsoncpp\include;..\third_party\blake3\c;..\third_party\googletest\googletest\include;..\third_party\protobuf\src;..\third_party\grpc\include;..\bazel-out\x64_windows-dbg\bin;$(VC_IncludePath);$(WindowsSDK_IncludePath)</BazelIncludePaths>
+    <BazelSourcePathPrefix>..\/</BazelSourcePathPrefix>
+  </PropertyGroup>
+  <Import Project="..\NMakeBazelProject.targets" />
+  <!-- For some reason, msbuild doesn't include this file, so copy it explicitly. -->
+  <!-- TODO: Reenable once we can cross-compile these.
+  <PropertyGroup>
+    <CdcFuseFsFile>$(SolutionDir)bazel-out\k8-$(BazelCompilationMode)\bin\cdc_fuse_fs\cdc_fuse_fs</CdcFuseFsFile>
+    <LibFuseFile>$(SolutionDir)bazel-out\k8-$(BazelCompilationMode)\bin\third_party\fuse\libfuse.so</LibFuseFile>
+  </PropertyGroup>
+  <Target Name="CopyCdcFuseFs" Inputs="$(CdcFuseFsFile)" Outputs="$(OutDir)cdc_fuse_fs" AfterTargets="Build">
+    <Copy SourceFiles="$(CdcFuseFsFile)" DestinationFiles="$(OutDir)cdc_fuse_fs" />
+  </Target>
+  <Target Name="CopyLibFuse" Inputs="$(LibFuseFile)" Outputs="$(OutDir)libfuse.so" AfterTargets="Build">
+    <Copy SourceFiles="$(LibFuseFile)" DestinationFiles="$(OutDir)libfuse.so" />
+  </Target> -->
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
+  <ImportGroup Label="ExtensionTargets">
+  </ImportGroup>
+</Project>

asset_stream_manager/asset_stream_manager.vcxproj.filters

@@ -0,0 +1,2 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003" />

asset_stream_manager/asset_stream_server.cc

@@ -0,0 +1,41 @@
+// Copyright 2022 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "asset_stream_manager/asset_stream_server.h"
+
+#include "asset_stream_manager/grpc_asset_stream_server.h"
+#include "asset_stream_manager/testing_asset_stream_server.h"
+
+namespace cdc_ft {
+
+AssetStreamServer::AssetStreamServer(std::string src_dir,
+                                     DataStoreReader* data_store_reader,
+                                     FileChunkMap* file_chunks) {}
+
+std::unique_ptr<AssetStreamServer> AssetStreamServer::Create(
+    AssetStreamServerType type, std::string src_dir,
+    DataStoreReader* data_store_reader, FileChunkMap* file_chunks,
+    ContentSentHandler content_sent) {
+  switch (type) {
+    case AssetStreamServerType::kGrpc:
+      return std::make_unique<GrpcAssetStreamServer>(src_dir, data_store_reader,
+                                                     file_chunks, content_sent);
+    case AssetStreamServerType::kTest:
+      return std::make_unique<TestingAssetStreamServer>(
+          src_dir, data_store_reader, file_chunks);
+  }
+  assert(false);
+  return nullptr;
+}
+}  // namespace cdc_ft

asset_stream_manager/asset_stream_server.h

@@ -0,0 +1,91 @@
+/*
+ * Copyright 2022 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef ASSET_STREAM_MANAGER_ASSET_STREAM_SERVER_H_
+#define ASSET_STREAM_MANAGER_ASSET_STREAM_SERVER_H_
+#include <memory>
+#include <string>
+
+#include "absl/status/status.h"
+#include "absl/time/time.h"
+#include "manifest/manifest_proto_defs.h"
+
+namespace cdc_ft {
+
+// Handles an event when content is transmitted from the workstation to a
+// gamelet.
+// |byte_count| number of bytes transferred during the session so far.
+// |chunk_count| number of chunks transferred during the session so far.
+// |instance_id| instance id, which identifies the session.
+using ContentSentHandler = std::function<void(
+    size_t byte_count, size_t chunk_count, std::string instance_id)>;
+
+class DataStoreReader;
+class FileChunkMap;
+
+enum class AssetStreamServerType { kGrpc, kTest };
+
+class AssetStreamServer {
+ public:
+  // Returns AssetStreamServer of |type|.
+  // |src_dir| is the directory on the workstation to mount.
+  // |data_store_reader| is responsible for loading content by ID.
+  // |file_chunks| is used for mapping data chunk ids to file locations.
+  // |content_sent| handles event when data is transferred from the workstation
+  // to a gamelet.
+  static std::unique_ptr<AssetStreamServer> Create(
+      AssetStreamServerType type, std::string src_dir,
+      DataStoreReader* data_store_reader, FileChunkMap* file_chunks,
+      ContentSentHandler content_sent);
+
+  AssetStreamServer(const AssetStreamServer& other) = delete;
+  AssetStreamServer& operator=(const AssetStreamServer& other) = delete;
+  virtual ~AssetStreamServer() = default;
+
+  // Starts the asset stream server on the given |port|.
+  // Asserts that the server is not yet running.
+  virtual absl::Status Start(int port) = 0;
+
+  // Sets |manifest_id| to be distributed to gamelets.
+  // Thread-safe.
+  virtual void SetManifestId(const ContentIdProto& manifest_id) = 0;
+
+  // Waits until the FUSE for the given |instance| id has acknowledged the
+  // reception of the currently set manifest id. Returns a DeadlineExceeded
+  // error if the ack is not received within the given |timeout|.
+  // Thread-safe.
+  virtual absl::Status WaitForManifestAck(const std::string& instance,
+                                          absl::Duration timeout) = 0;
+
+  // Stops internal services and waits for the server to shut down.
+  virtual void Shutdown() = 0;
+
+  // Returns the used manifest id.
+  // Thread-safe.
+  virtual ContentIdProto GetManifestId() const = 0;
+
+ protected:
+  // Creates a new asset streaming server.
+  // |src_dir| is the directory on the workstation to mount.
+  // |data_store_reader| is responsible for loading content by ID.
+  // |file_chunks| is used for mapping data chunk ids to file locations.
+  AssetStreamServer(std::string src_dir, DataStoreReader* data_store_reader,
+                    FileChunkMap* file_chunks);
+};
+
+}  // namespace cdc_ft
+
+#endif  // ASSET_STREAM_MANAGER_ASSET_STREAM_SERVER_H_

asset_stream_manager/background_service_impl.cc

@@ -0,0 +1,56 @@
+// Copyright 2022 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "asset_stream_manager/background_service_impl.h"
+
+#include "common/grpc_status.h"
+#include "common/log.h"
+#include "common/util.h"
+#include "grpcpp/grpcpp.h"
+
+namespace cdc_ft {
+
+BackgroundServiceImpl::BackgroundServiceImpl() {}
+
+BackgroundServiceImpl::~BackgroundServiceImpl() = default;
+
+void BackgroundServiceImpl::SetExitCallback(ExitCallback exit_callback) {
+  exit_callback_ = std::move(exit_callback);
+}
+
+grpc::Status BackgroundServiceImpl::Exit(grpc::ServerContext* context,
+                                         const ExitRequest* request,
+                                         ExitResponse* response) {
+  LOG_INFO("RPC:Exit");
+  if (exit_callback_) {
+    return ToGrpcStatus(exit_callback_());
+  }
+  return grpc::Status::OK;
+}
+
+grpc::Status BackgroundServiceImpl::GetPid(grpc::ServerContext* context,
+                                           const GetPidRequest* request,
+                                           GetPidResponse* response) {
+  LOG_INFO("RPC:GetPid");
+  response->set_pid(static_cast<int32_t>(Util::GetPid()));
+  return grpc::Status::OK;
+}
+
+grpc::Status BackgroundServiceImpl::HealthCheck(grpc::ServerContext* context,
+                                                const EmptyProto* request,
+                                                EmptyProto* response) {
+  return grpc::Status::OK;
+}
+
+}  // namespace cdc_ft

asset_stream_manager/background_service_impl.h

@@ -0,0 +1,68 @@
+/*
+ * Copyright 2022 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef ASSET_STREAM_MANAGER_BACKGROUND_SERVICE_IMPL_H_
+#define ASSET_STREAM_MANAGER_BACKGROUND_SERVICE_IMPL_H_
+
+#include "absl/status/status.h"
+#include "asset_stream_manager/background_service_impl.h"
+#include "asset_stream_manager/session_management_server.h"
+#include "grpcpp/grpcpp.h"
+#include "proto/background_service.grpc.pb.h"
+
+namespace cdc_ft {
+
+// Implements a service to manage a background process as a server.
+// The corresponding client is implemented by ProcessManager. The background
+// process in this case is asset_stream_manager. ProcessManager starts the
+// process on demand (e.g. when `ggp instance mount --local-dir` is invoked) and
+// manages its lifetime: It calls GetPid() initially, HealthCheck() periodically
+// to monitor the process, and Exit() on shutdown.
+// This service is owned by SessionManagementServer.
+class BackgroundServiceImpl final
+    : public backgroundservice::BackgroundService::Service {
+ public:
+  using ExitRequest = backgroundservice::ExitRequest;
+  using ExitResponse = backgroundservice::ExitResponse;
+  using GetPidRequest = backgroundservice::GetPidRequest;
+  using GetPidResponse = backgroundservice::GetPidResponse;
+  using EmptyProto = google::protobuf::Empty;
+
+  BackgroundServiceImpl();
+  ~BackgroundServiceImpl();
+
+  // Exit callback gets called from the Exit() RPC.
+  using ExitCallback = std::function<absl::Status()>;
+  void SetExitCallback(ExitCallback exit_callback);
+
+  grpc::Status Exit(grpc::ServerContext* context, const ExitRequest* request,
+                    ExitResponse* response) override;
+
+  grpc::Status GetPid(grpc::ServerContext* context,
+                      const GetPidRequest* request,
+                      GetPidResponse* response) override;
+
+  grpc::Status HealthCheck(grpc::ServerContext* context,
+                           const EmptyProto* request,
+                           EmptyProto* response) override;
+
+ private:
+  ExitCallback exit_callback_;
+};
+
+}  // namespace cdc_ft
+
+#endif  // ASSET_STREAM_MANAGER_BACKGROUND_SERVICE_IMPL_H_

asset_stream_manager/cdc_fuse_manager.cc

@@ -0,0 +1,225 @@
+// Copyright 2022 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "asset_stream_manager/cdc_fuse_manager.h"
+
+#include "absl/strings/match.h"
+#include "absl/strings/str_format.h"
+#include "cdc_fuse_fs/constants.h"
+#include "common/gamelet_component.h"
+#include "common/log.h"
+#include "common/path.h"
+#include "common/status.h"
+#include "common/status_macros.h"
+
+namespace cdc_ft {
+namespace {
+
+constexpr char kFuseFilename[] = "cdc_fuse_fs";
+constexpr char kLibFuseFilename[] = "libfuse.so";
+constexpr char kFuseStdoutPrefix[] = "cdc_fuse_fs_stdout";
+constexpr char kRemoteToolsBinDir[] = "/opt/developer/tools/bin/";
+
+// Mount point for FUSE on the gamelet.
+constexpr char kMountDir[] = "/mnt/workstation";
+
+// Cache directory on the gamelet to store data chunks.
+constexpr char kCacheDir[] = "/var/cache/asset_streaming";
+
+}  // namespace
+
+CdcFuseManager::CdcFuseManager(std::string instance,
+                               ProcessFactory* process_factory,
+                               RemoteUtil* remote_util)
+    : instance_(std::move(instance)),
+      process_factory_(process_factory),
+      remote_util_(remote_util) {}
+
+CdcFuseManager::~CdcFuseManager() = default;
+
+absl::Status CdcFuseManager::Deploy() {
+  assert(!fuse_process_);
+
+  LOG_INFO("Deploying FUSE...");
+
+  std::string exe_dir;
+  RETURN_IF_ERROR(path::GetExeDir(&exe_dir), "Failed to get exe directory");
+
+  std::string local_exe_path = path::Join(exe_dir, kFuseFilename);
+  std::string local_lib_path = path::Join(exe_dir, kLibFuseFilename);
+
+#ifdef _DEBUG
+  // Sync FUSE to the gamelet in debug. Debug builds are rather large, so
+  // there's a gain from using sync.
+  LOG_DEBUG("Syncing FUSE");
+  RETURN_IF_ERROR(
+      remote_util_->Sync({local_exe_path, local_lib_path}, kRemoteToolsBinDir),
+      "Failed to sync FUSE to gamelet");
+  LOG_DEBUG("Syncing FUSE succeeded");
+#else
+  // Copy FUSE to the gamelet. This is usually faster in production since it
+  // doesn't have to deploy ggp__server first.
+  LOG_DEBUG("Copying FUSE");
+  RETURN_IF_ERROR(remote_util_->Scp({local_exe_path, local_lib_path},
+                                    kRemoteToolsBinDir, true),
+                  "Failed to copy FUSE to gamelet");
+  LOG_DEBUG("Copying FUSE succeeded");
+
+  // Make FUSE executable. Note that sync does it automatically.
+  LOG_DEBUG("Making FUSE executable");
+  std::string remotePath = path::JoinUnix(kRemoteToolsBinDir, kFuseFilename);
+  RETURN_IF_ERROR(remote_util_->Chmod("a+x", remotePath),
+                  "Failed to set executable flag on FUSE");
+  LOG_DEBUG("Making FUSE succeeded");
+#endif
+
+  return absl::OkStatus();
+}
+
+absl::Status CdcFuseManager::Start(uint16_t local_port, uint16_t remote_port,
+                                   int verbosity, bool debug,
+                                   bool singlethreaded, bool enable_stats,
+                                   bool check, uint64_t cache_capacity,
+                                   uint32_t cleanup_timeout_sec,
+                                   uint32_t access_idle_timeout_sec) {
+  assert(!fuse_process_);
+
+  // Gather stats for the FUSE gamelet component to determine whether a
+  // re-deploy is necessary.
+  std::string exe_dir;
+  RETURN_IF_ERROR(path::GetExeDir(&exe_dir), "Failed to get exe directory");
+  std::vector<GameletComponent> components;
+  absl::Status status =
+      GameletComponent::Get({path::Join(exe_dir, kFuseFilename),
+                             path::Join(exe_dir, kLibFuseFilename)},
+                            &components);
+  if (!status.ok()) {
+    return absl::NotFoundError(absl::StrFormat(
+        "Required gamelet component not found. Make sure the files %s and %s "
+        "reside in the same folder as stadia_assets_stream_manager_v3.exe.",
+        kFuseFilename, kLibFuseFilename));
+  }
+  std::string component_args = GameletComponent::ToCommandLineArgs(components);
+
+  // Build the remote command.
+  std::string remotePath = path::JoinUnix(kRemoteToolsBinDir, kFuseFilename);
+  std::string remote_command = absl::StrFormat(
+      "LD_LIBRARY_PATH=%s %s --instance='%s' "
+      "--components='%s' --port=%i --cache_dir=%s "
+      "--verbosity=%i --cleanup_timeout=%i --access_idle_timeout=%i --stats=%i "
+      "--check=%i --cache_capacity=%u -- -o allow_root -o ro -o nonempty -o "
+      "auto_unmount %s%s%s",
+      kRemoteToolsBinDir, remotePath, instance_, component_args, remote_port,
+      kCacheDir, verbosity, cleanup_timeout_sec, access_idle_timeout_sec,
+      enable_stats, check, cache_capacity, kMountDir, debug ? " -d" : "",
+      singlethreaded ? " -s" : "");
+
+  bool needs_deploy = false;
+  RETURN_IF_ERROR(
+      RunFuseProcess(local_port, remote_port, remote_command, &needs_deploy));
+  if (needs_deploy) {
+    // Deploy and try again.
+    RETURN_IF_ERROR(Deploy());
+    RETURN_IF_ERROR(
+        RunFuseProcess(local_port, remote_port, remote_command, &needs_deploy));
+  }
+
+  return absl::OkStatus();
+}
+
+absl::Status CdcFuseManager::RunFuseProcess(uint16_t local_port,
+                                            uint16_t remote_port,
+                                            const std::string& remote_command,
+                                            bool* needs_deploy) {
+  assert(!fuse_process_);
+  assert(needs_deploy);
+  *needs_deploy = false;
+
+  LOG_DEBUG("Running FUSE process");
+  ProcessStartInfo start_info =
+      remote_util_->BuildProcessStartInfoForSshPortForwardAndCommand(
+          local_port, remote_port, true, remote_command);
+  start_info.name = kFuseFilename;
+
+  // Capture stdout to determine whether a deploy is required.
+  fuse_stdout_.clear();
+  fuse_startup_finished_ = false;
+  start_info.stdout_handler = [this, needs_deploy](const char* data,
+                                                   size_t size) {
+    return HandleFuseStdout(data, size, needs_deploy);
+  };
+  fuse_process_ = process_factory_->Create(start_info);
+  RETURN_IF_ERROR(fuse_process_->Start(), "Failed to start FUSE process");
+  LOG_DEBUG("FUSE process started. Waiting for startup to finish.");
+
+  // Run until process exits or startup finishes.
+  auto startup_finished = [this]() { return fuse_startup_finished_.load(); };
+  RETURN_IF_ERROR(fuse_process_->RunUntil(startup_finished),
+                  "Failed to run FUSE process");
+  LOG_DEBUG("FUSE process startup complete.");
+
+  // If the FUSE process exited before it could perform its up-to-date check, it
+  // most likely happens because the binary does not exist and needs to be
+  // deployed.
+  *needs_deploy |= !fuse_startup_finished_ && fuse_process_->HasExited() &&
+                   fuse_process_->ExitCode() != 0;
+  if (*needs_deploy) {
+    LOG_DEBUG("FUSE needs to be (re-)deployed.");
+    fuse_process_.reset();
+    return absl::OkStatus();
+  }
+
+  return absl::OkStatus();
+}
+
+absl::Status CdcFuseManager::Stop() {
+  if (!fuse_process_) {
+    return absl::OkStatus();
+  }
+
+  LOG_DEBUG("Terminating FUSE process");
+  absl::Status status = fuse_process_->Terminate();
+  fuse_process_.reset();
+  return status;
+}
+
+bool CdcFuseManager::IsHealthy() const {
+  return fuse_process_ && !fuse_process_->HasExited();
+}
+
+absl::Status CdcFuseManager::HandleFuseStdout(const char* data, size_t size,
+                                              bool* needs_deploy) {
+  assert(needs_deploy);
+
+  // Don't capture stdout beyond startup.
+  if (!fuse_startup_finished_) {
+    fuse_stdout_.append(data, size);
+    // The gamelet component prints some magic strings to stdout to indicate
+    // whether it's up-to-date.
+    if (absl::StrContains(fuse_stdout_, kFuseUpToDate)) {
+      fuse_startup_finished_ = true;
+    } else if (absl::StrContains(fuse_stdout_, kFuseNotUpToDate)) {
+      fuse_startup_finished_ = true;
+      *needs_deploy = true;
+    }
+  }
+
+  if (!remote_util_->Quiet()) {
+    // Forward to logging.
+    return LogOutput(kFuseStdoutPrefix, data, size);
+  }
+  return absl::OkStatus();
+}
+
+}  // namespace cdc_ft

asset_stream_manager/cdc_fuse_manager.h

@@ -0,0 +1,98 @@
+/*
+ * Copyright 2022 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef ASSET_STREAM_MANAGER_CDC_FUSE_MANAGER_H_
+#define ASSET_STREAM_MANAGER_CDC_FUSE_MANAGER_H_
+
+#include "absl/status/status.h"
+#include "common/remote_util.h"
+
+namespace cdc_ft {
+
+class Process;
+class ProcessFactory;
+class RemoteUtil;
+
+// Manages the gamelet-side CDC FUSE filesystem process.
+class CdcFuseManager {
+ public:
+  CdcFuseManager(std::string instance, ProcessFactory* process_factory,
+                 RemoteUtil* remote_util);
+  ~CdcFuseManager();
+
+  CdcFuseManager(CdcFuseManager&) = delete;
+  CdcFuseManager& operator=(CdcFuseManager&) = delete;
+
+  // Starts the CDC FUSE and establishes a reverse SSH tunnel from the gamelet's
+  // |remote_port| to the workstation's |local_port|. Deploys the binary if
+  // necessary.
+  //
+  // |verbosity| is the log verbosity used by the filesystem.
+  // |debug| puts the filesystem into debug mode if set to true. This also
+  // causes the process to run in the foreground, so that logs are piped through
+  // SSH to stdout of the workstation process.
+  // |singlethreaded| puts the filesystem into single-threaded mode if true.
+  // |enable_stats| determines whether FUSE should send debug statistics.
+  // |check| determines whether to execute FUSE consistency check.
+  // |cache_capacity| defines the cache capacity in bytes.
+  // |cleanup_timeout_sec| defines the data provider cleanup timeout in seconds.
+  // |access_idle_timeout_sec| defines the number of seconds after which data
+  // provider is considered to be access-idling.
+  absl::Status Start(uint16_t local_port, uint16_t remote_port, int verbosity,
+                     bool debug, bool singlethreaded, bool enable_stats,
+                     bool check, uint64_t cache_capacity,
+                     uint32_t cleanup_timeout_sec,
+                     uint32_t access_idle_timeout_sec);
+
+  // Stops the CDC FUSE.
+  absl::Status Stop();
+
+  // Returns true if the FUSE process is running.
+  bool IsHealthy() const;
+
+ private:
+  // Runs the FUSE process on the gamelet from the given |remote_command| and
+  // establishes a reverse SSH tunnel from the gamelet's |remote_port| to the
+  // workstation's |local_port|.
+  //
+  // If the FUSE is not up-to-date or does not exist, sets |needs_deploy| to
+  // true and returns OK. In that case, Deploy() needs to be called and the FUSE
+  // process should be run again.
+  absl::Status RunFuseProcess(uint16_t local_port, uint16_t remote_port,
+                              const std::string& remote_command,
+                              bool* needs_deploy);
+
+  // Deploys the gamelet components.
+  absl::Status Deploy();
+
+  // Output handler for FUSE's stdout. Sets |needs_deploy| to true if the output
+  // contains a magic marker to indicate that the binary has to be redeployed.
+  // Called in a background thread.
+  absl::Status HandleFuseStdout(const char* data, size_t size,
+                                bool* needs_deploy);
+
+  std::string instance_;
+  ProcessFactory* const process_factory_;
+  RemoteUtil* const remote_util_;
+
+  std::unique_ptr<Process> fuse_process_;
+  std::string fuse_stdout_;
+  std::atomic<bool> fuse_startup_finished_{false};
+};
+
+}  // namespace cdc_ft
+
+#endif  // ASSET_STREAM_MANAGER_CDC_FUSE_MANAGER_H_

asset_stream_manager/grpc_asset_stream_server.cc

@@ -0,0 +1,305 @@
+// Copyright 2022 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "asset_stream_manager/grpc_asset_stream_server.h"
+
+#include "absl/strings/str_format.h"
+#include "absl/time/time.h"
+#include "common/grpc_status.h"
+#include "common/log.h"
+#include "common/path.h"
+#include "common/status.h"
+#include "common/status_macros.h"
+#include "data_store/data_store_reader.h"
+#include "grpcpp/grpcpp.h"
+#include "manifest/file_chunk_map.h"
+#include "proto/asset_stream_service.grpc.pb.h"
+
+namespace cdc_ft {
+namespace {
+
+using GetContentRequest = proto::GetContentRequest;
+using GetContentResponse = proto::GetContentResponse;
+using SendCachedContentIdsRequest = proto::SendCachedContentIdsRequest;
+using SendCachedContentIdsResponse = proto::SendCachedContentIdsResponse;
+using AssetStreamService = proto::AssetStreamService;
+
+using GetManifestIdRequest = proto::GetManifestIdRequest;
+using GetManifestIdResponse = proto::GetManifestIdResponse;
+using AckManifestIdReceivedRequest = proto::AckManifestIdReceivedRequest;
+using AckManifestIdReceivedResponse = proto::AckManifestIdReceivedResponse;
+using ConfigStreamService = proto::ConfigStreamService;
+
+}  // namespace
+
+class AssetStreamServiceImpl final : public AssetStreamService::Service {
+ public:
+  AssetStreamServiceImpl(std::string src_dir,
+                         DataStoreReader* data_store_reader,
+                         FileChunkMap* file_chunks, InstanceIdMap* instance_ids,
+                         ContentSentHandler content_sent)
+      : src_dir_(std::move(src_dir)),
+        data_store_reader_(data_store_reader),
+        file_chunks_(file_chunks),
+        started_(absl::Now()),
+        instance_ids_(instance_ids),
+        content_sent_(content_sent) {}
+
+  grpc::Status GetContent(grpc::ServerContext* context,
+                          const GetContentRequest* request,
+                          GetContentResponse* response) override {
+    // See if this is a data chunk first. The hash lookup is faster than the
+    // file lookup from the data store.
+    std::string rel_path;
+    uint64_t offset;
+    size_t size;
+    for (const ContentIdProto& id : request->id()) {
+      uint32_t uint32_size;
+      if (file_chunks_->Lookup(id, &rel_path, &offset, &uint32_size)) {
+        size = uint32_size;
+        // File data chunk.
+        RETURN_GRPC_IF_ERROR(ReadFromFile(id, rel_path, offset, uint32_size,
+                                          response->add_data()));
+        file_chunks_->RecordStreamedChunk(id, request->thread_id());
+      } else {
+        // Manifest chunk.
+        RETURN_GRPC_IF_ERROR(
+            ReadFromDataStore(id, response->add_data(), &size));
+      }
+      std::string instance_id = instance_ids_->Get(context->peer());
+      if (content_sent_ != nullptr) {
+        content_sent_(size, 1, instance_id);
+      }
+    }
+    return grpc::Status::OK;
+  }
+
+  grpc::Status SendCachedContentIds(
+      grpc::ServerContext* context, const SendCachedContentIdsRequest* request,
+      SendCachedContentIdsResponse* response) override {
+    for (const ContentIdProto& id : request->id())
+      file_chunks_->RecordCachedChunk(id);
+    return grpc::Status::OK;
+  }
+
+ private:
+  absl::Status ReadFromFile(const ContentIdProto& id,
+                            const std::string& rel_path, uint64_t offset,
+                            uint32_t size, std::string* data) {
+    std::string path = path::Join(src_dir_, rel_path);
+    path::FixPathSeparators(&path);
+    data->resize(size);
+    size_t read_size;
+    ASSIGN_OR_RETURN(
+        read_size,
+        path::ReadFile(path, const_cast<char*>(data->data()), offset, size),
+        "Failed to read chunk '%s', file '%s', offset %d, size %d",
+        ContentId::ToHexString(id), path, offset, size);
+
+    absl::Time now = absl::Now();
+    LOG_VERBOSE("'%s', %d, '%s', '%s', %u, %u",
+                absl::FormatTime("%H:%M:%S", now, absl::UTCTimeZone()),
+                absl::ToInt64Milliseconds(now - started_),
+                ContentId::ToHexString(id), path, offset, size);
+
+    return absl::OkStatus();
+  }
+
+  absl::Status ReadFromDataStore(const ContentIdProto& id, std::string* data,
+                                 size_t* size) {
+    Buffer buf;
+    RETURN_IF_ERROR(data_store_reader_->Get(id, &buf),
+                    "Failed to read chunk '%s'", ContentId::ToHexString(id));
+
+    // TODO: Get rid of copy after the Buffer uses std::string.
+    *data = std::string(buf.data(), buf.size());
+    *size = buf.size();
+    absl::Time now = absl::Now();
+    LOG_VERBOSE("'%s', %d, '%s', %d",
+                absl::FormatTime("%H:%M:%S", now, absl::UTCTimeZone()),
+                absl::ToInt64Milliseconds(now - started_),
+                ContentId::ToHexString(id), buf.size());
+
+    return absl::OkStatus();
+  }
+
+  const std::string src_dir_;
+  DataStoreReader* const data_store_reader_;
+  FileChunkMap* const file_chunks_;
+  const absl::Time started_;
+  InstanceIdMap* instance_ids_;
+  ContentSentHandler content_sent_;
+};
+
+class ConfigStreamServiceImpl final : public ConfigStreamService::Service {
+ public:
+  explicit ConfigStreamServiceImpl(InstanceIdMap* instance_ids)
+      : instance_ids_(instance_ids) {}
+  ~ConfigStreamServiceImpl() { Shutdown(); }
+
+  grpc::Status GetManifestId(
+      grpc::ServerContext* context, const GetManifestIdRequest* request,
+      ::grpc::ServerWriter<GetManifestIdResponse>* stream) override {
+    ContentIdProto local_id;
+    bool running = true;
+    do {
+      // Shutdown happened.
+      if (!WaitForUpdate(local_id)) {
+        break;
+      }
+      LOG_INFO("Sending updated manifest id '%s' to the gamelet",
+               ContentId::ToHexString(local_id));
+      GetManifestIdResponse response;
+      *response.mutable_id() = local_id;
+      bool success = stream->Write(response);
+      if (!success) {
+        LOG_WARNING("Failed to send updated manifest id '%s'",
+                    ContentId::ToHexString(local_id));
+      }
+      absl::ReaderMutexLock lock(&mutex_);
+      running = running_;
+    } while (running);
+    return grpc::Status::OK;
+  }
+
+  grpc::Status AckManifestIdReceived(
+      grpc::ServerContext* context, const AckManifestIdReceivedRequest* request,
+      AckManifestIdReceivedResponse* response) override {
+    // Associate the peer with the gamelet ID.
+    instance_ids_->Set(context->peer(), request->gamelet_id());
+    absl::MutexLock lock(&mutex_);
+    acked_manifest_ids_[request->gamelet_id()] = request->manifest_id();
+    return grpc::Status::OK;
+  }
+
+  void SetManifestId(const ContentIdProto& id) ABSL_LOCKS_EXCLUDED(mutex_) {
+    LOG_INFO("Updating manifest id '%s' in configuration service",
+             ContentId::ToHexString(id));
+    absl::MutexLock lock(&mutex_);
+    id_ = id;
+  }
+
+  absl::Status WaitForManifestAck(const std::string& instance,
+                                  absl::Duration timeout) {
+    absl::MutexLock lock(&mutex_);
+    auto cond = [this, &instance]() ABSL_EXCLUSIVE_LOCKS_REQUIRED(mutex_) {
+      AckedManifestIdsMap::iterator iter = acked_manifest_ids_.find(instance);
+      return iter != acked_manifest_ids_.end() && id_ == iter->second;
+    };
+
+    if (!mutex_.AwaitWithTimeout(absl::Condition(&cond), timeout)) {
+      return absl::DeadlineExceededError(absl::StrFormat(
+          "Instance '%s' did not acknowledge reception of manifest", instance));
+    }
+
+    return absl::OkStatus();
+  }
+
+  void Shutdown() ABSL_LOCKS_EXCLUDED(mutex_) {
+    absl::MutexLock lock(&mutex_);
+    if (running_) {
+      LOG_INFO("Shutting down configuration service");
+      running_ = false;
+    }
+  }
+
+  ContentIdProto GetStoredManifestId() const ABSL_LOCKS_EXCLUDED(mutex_) {
+    absl::MutexLock lock(&mutex_);
+    return id_;
+  }
+
+ private:
+  // Returns false if the update process was cancelled.
+  bool WaitForUpdate(ContentIdProto& local_id) ABSL_LOCKS_EXCLUDED(mutex_) {
+    absl::MutexLock lock(&mutex_);
+    auto cond = [&]() ABSL_EXCLUSIVE_LOCKS_REQUIRED(mutex_) {
+      return !running_ || local_id != id_;
+    };
+    mutex_.Await(absl::Condition(&cond));
+    local_id = id_;
+    return running_;
+  }
+
+  mutable absl::Mutex mutex_;
+  ContentIdProto id_ ABSL_GUARDED_BY(mutex_);
+  bool running_ ABSL_GUARDED_BY(mutex_) = true;
+  InstanceIdMap* instance_ids_;
+
+  // Maps instance ids to the last acknowledged manifest id.
+  using AckedManifestIdsMap = std::unordered_map<std::string, ContentIdProto>;
+  AckedManifestIdsMap acked_manifest_ids_ ABSL_GUARDED_BY(mutex_);
+};
+
+GrpcAssetStreamServer::GrpcAssetStreamServer(std::string src_dir,
+                                             DataStoreReader* data_store_reader,
+                                             FileChunkMap* file_chunks,
+                                             ContentSentHandler content_sent)
+    : AssetStreamServer(src_dir, data_store_reader, file_chunks),
+      asset_stream_service_(std::make_unique<AssetStreamServiceImpl>(
+          std::move(src_dir), data_store_reader, file_chunks, &instance_ids_,
+          content_sent)),
+      config_stream_service_(
+          std::make_unique<ConfigStreamServiceImpl>(&instance_ids_)) {}
+
+GrpcAssetStreamServer::~GrpcAssetStreamServer() = default;
+
+absl::Status GrpcAssetStreamServer::Start(int port) {
+  assert(!server_);
+
+  std::string server_address = absl::StrFormat("localhost:%i", port);
+  grpc::ServerBuilder builder;
+  int selected_port = 0;
+  builder.AddListeningPort(server_address, grpc::InsecureServerCredentials(),
+                           &selected_port);
+  builder.RegisterService(asset_stream_service_.get());
+  builder.RegisterService(config_stream_service_.get());
+  server_ = builder.BuildAndStart();
+  if (selected_port != port) {
+    return MakeStatus(
+        "Failed to start streaming server: Could not listen on port %i. Is the "
+        "port in use?",
+        port);
+  }
+  if (!server_) return MakeStatus("Failed to start streaming server");
+  LOG_INFO("Streaming server listening on '%s'", server_address);
+  return absl::OkStatus();
+}
+
+void GrpcAssetStreamServer::SetManifestId(const ContentIdProto& manifest_id) {
+  LOG_INFO("Setting manifest id '%s'", ContentId::ToHexString(manifest_id));
+  assert(config_stream_service_);
+  config_stream_service_->SetManifestId(manifest_id);
+}
+
+absl::Status GrpcAssetStreamServer::WaitForManifestAck(
+    const std::string& instance, absl::Duration timeout) {
+  assert(config_stream_service_);
+  return config_stream_service_->WaitForManifestAck(instance, timeout);
+}
+
+void GrpcAssetStreamServer::Shutdown() {
+  assert(config_stream_service_);
+  config_stream_service_->Shutdown();
+  if (server_) {
+    server_->Shutdown();
+    server_->Wait();
+  }
+}
+
+ContentIdProto GrpcAssetStreamServer::GetManifestId() const {
+  assert(config_stream_service_);
+  return config_stream_service_->GetStoredManifestId();
+}
+
+}  // namespace cdc_ft

asset_stream_manager/grpc_asset_stream_server.h

@@ -0,0 +1,69 @@
+/*
+ * Copyright 2022 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef ASSET_STREAM_MANAGER_GRPC_ASSET_STREAM_SERVER_H_
+#define ASSET_STREAM_MANAGER_GRPC_ASSET_STREAM_SERVER_H_
+
+#include <memory>
+#include <string>
+
+#include "asset_stream_manager/asset_stream_server.h"
+#include "common/thread_safe_map.h"
+
+namespace grpc {
+class Server;
+}
+
+namespace cdc_ft {
+
+using InstanceIdMap = ThreadSafeMap<std::string, std::string>;
+
+class AssetStreamServiceImpl;
+class ConfigStreamServiceImpl;
+
+// gRpc server for streaming assets to one or more gamelets.
+class GrpcAssetStreamServer : public AssetStreamServer {
+ public:
+  // Creates a new asset streaming gRpc server.
+  GrpcAssetStreamServer(std::string src_dir, DataStoreReader* data_store_reader,
+                        FileChunkMap* file_chunks,
+                        ContentSentHandler content_sent);
+
+  ~GrpcAssetStreamServer();
+
+  // AssetStreamServer:
+
+  absl::Status Start(int port) override;
+
+  void SetManifestId(const ContentIdProto& manifest_id) override;
+
+  absl::Status WaitForManifestAck(const std::string& instance,
+                                  absl::Duration timeout) override;
+
+  void Shutdown() override;
+
+  ContentIdProto GetManifestId() const override;
+
+ private:
+  InstanceIdMap instance_ids_;
+  const std::unique_ptr<AssetStreamServiceImpl> asset_stream_service_;
+  const std::unique_ptr<ConfigStreamServiceImpl> config_stream_service_;
+  std::unique_ptr<grpc::Server> server_;
+};
+
+}  // namespace cdc_ft
+
+#endif  // ASSET_STREAM_MANAGER_GRPC_ASSET_STREAM_SERVER_H_

asset_stream_manager/local_assets_stream_manager_service_impl.cc

@@ -0,0 +1,259 @@
+// Copyright 2022 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "asset_stream_manager/local_assets_stream_manager_service_impl.h"
+
+#include <iomanip>
+
+#include "absl/strings/str_format.h"
+#include "absl/strings/str_split.h"
+#include "asset_stream_manager/multi_session.h"
+#include "asset_stream_manager/session_manager.h"
+#include "common/grpc_status.h"
+#include "common/log.h"
+#include "common/path.h"
+#include "common/process.h"
+#include "common/sdk_util.h"
+#include "common/status.h"
+#include "manifest/manifest_updater.h"
+
+namespace cdc_ft {
+namespace {
+
+// Parses |instance_name| of the form
+// "organizations/{org-id}/projects/{proj-id}/pools/{pool-id}/gamelets/{gamelet-id}"
+// into parts. The pool id is not returned.
+bool ParseInstanceName(const std::string& instance_name,
+                       std::string* instance_id, std::string* project_id,
+                       std::string* organization_id) {
+  std::string pool_id;
+  std::vector<std::string> parts = absl::StrSplit(instance_name, '/');
+  if (parts.size() != 10) return false;
+  if (parts[0] != "organizations" || parts[1].empty()) return false;
+  if (parts[2] != "projects" || parts[3].empty()) return false;
+  if (parts[4] != "pools" || parts[5].empty()) return false;
+  // Instance id is e.g.
+  // edge/e-europe-west3-b/49d010c7be1845ac9a19a9033c64a460ces1
+  if (parts[6] != "gamelets" || parts[7].empty() || parts[8].empty() ||
+      parts[9].empty())
+    return false;
+  *organization_id = parts[1];
+  *project_id = parts[3];
+  *instance_id = absl::StrFormat("%s/%s/%s", parts[7], parts[8], parts[9]);
+  return true;
+}
+
+// Parses |data| line by line for "|key|: value" and puts the first instance in
+// |value| if present. Returns false if |data| does not contain "|key|: value".
+// Trims whitespace.
+bool ParseValue(const std::string& data, const std::string& key,
+                std::string* value) {
+  std::istringstream stream(data);
+
+  std::string line;
+  while (std::getline(stream, line)) {
+    if (line.find(key + ":") == 0) {
+      // Trim value.
+      size_t start_pos = key.size() + 1;
+      while (start_pos < line.size() && isspace(line[start_pos])) {
+        start_pos++;
+      }
+      size_t end_pos = line.size();
+      while (end_pos > start_pos && isspace(line[end_pos - 1])) {
+        end_pos--;
+      }
+      *value = line.substr(start_pos, end_pos - start_pos);
+      return true;
+    }
+  }
+  return false;
+}
+
+// Why oh why?
+std::string Quoted(const std::string& s) {
+  std::ostringstream ss;
+  ss << std::quoted(s);
+  return ss.str();
+}
+
+}  // namespace
+
+LocalAssetsStreamManagerServiceImpl::LocalAssetsStreamManagerServiceImpl(
+    SessionManager* session_manager, ProcessFactory* process_factory,
+    metrics::MetricsService* metrics_service)
+    : session_manager_(session_manager),
+      process_factory_(process_factory),
+      metrics_service_(metrics_service) {}
+
+LocalAssetsStreamManagerServiceImpl::~LocalAssetsStreamManagerServiceImpl() =
+    default;
+
+grpc::Status LocalAssetsStreamManagerServiceImpl::StartSession(
+    grpc::ServerContext* /*context*/, const StartSessionRequest* request,
+    StartSessionResponse* /*response*/) {
+  LOG_INFO("RPC:StartSession(gamelet_name='%s', workstation_directory='%s'",
+           request->gamelet_name(), request->workstation_directory());
+
+  metrics::DeveloperLogEvent evt;
+  evt.as_manager_data = std::make_unique<metrics::AssetStreamingManagerData>();
+  evt.as_manager_data->session_start_data =
+      std::make_unique<metrics::SessionStartData>();
+  evt.as_manager_data->session_start_data->absl_status = absl::StatusCode::kOk;
+  evt.as_manager_data->session_start_data->status =
+      metrics::SessionStartStatus::kOk;
+  evt.as_manager_data->session_start_data->origin =
+      ConvertOrigin(request->origin());
+
+  // Parse instance/project/org id.
+  absl::Status status;
+  MultiSession* ms = nullptr;
+  std::string instance_id, project_id, organization_id, instance_ip;
+  uint16_t instance_port = 0;
+  if (!ParseInstanceName(request->gamelet_name(), &instance_id, &project_id,
+                         &organization_id)) {
+    status = absl::InvalidArgumentError(absl::StrFormat(
+        "Failed to parse instance name '%s'", request->gamelet_name()));
+  } else {
+    evt.project_id = project_id;
+    evt.organization_id = organization_id;
+
+    status = InitSsh(instance_id, project_id, organization_id, &instance_ip,
+                     &instance_port);
+
+    if (status.ok()) {
+      status = session_manager_->StartSession(
+          instance_id, project_id, organization_id, instance_ip, instance_port,
+          request->workstation_directory(), &ms,
+          &evt.as_manager_data->session_start_data->status);
+    }
+  }
+
+  evt.as_manager_data->session_start_data->absl_status = status.code();
+  if (ms) {
+    evt.as_manager_data->session_start_data->concurrent_session_count =
+        ms->GetSessionCount();
+    if (!instance_id.empty() && ms->HasSessionForInstance(instance_id)) {
+      ms->RecordSessionEvent(std::move(evt), metrics::EventType::kSessionStart,
+                             instance_id);
+    } else {
+      ms->RecordMultiSessionEvent(std::move(evt),
+                                  metrics::EventType::kSessionStart);
+    }
+  } else {
+    metrics_service_->RecordEvent(std::move(evt),
+                                  metrics::EventType::kSessionStart);
+  }
+
+  if (status.ok()) {
+    LOG_INFO("StartSession() succeeded");
+  } else {
+    LOG_ERROR("StartSession() failed: %s", status.ToString());
+  }
+  return ToGrpcStatus(status);
+}
+
+grpc::Status LocalAssetsStreamManagerServiceImpl::StopSession(
+    grpc::ServerContext* /*context*/, const StopSessionRequest* request,
+    StopSessionResponse* /*response*/) {
+  LOG_INFO("RPC:StopSession(gamelet_id='%s')", request->gamelet_id());
+
+  absl::Status status = session_manager_->StopSession(request->gamelet_id());
+  if (status.ok()) {
+    LOG_INFO("StopSession() succeeded");
+  } else {
+    LOG_ERROR("StopSession() failed: %s", status.ToString());
+  }
+  return ToGrpcStatus(status);
+}
+
+metrics::RequestOrigin LocalAssetsStreamManagerServiceImpl::ConvertOrigin(
+    StartSessionRequestOrigin origin) const {
+  switch (origin) {
+    case StartSessionRequest::ORIGIN_UNKNOWN:
+      return metrics::RequestOrigin::kUnknown;
+    case StartSessionRequest::ORIGIN_CLI:
+      return metrics::RequestOrigin::kCli;
+    case StartSessionRequest::ORIGIN_PARTNER_PORTAL:
+      return metrics::RequestOrigin::kPartnerPortal;
+    default:
+      return metrics::RequestOrigin::kUnknown;
+  }
+}
+
+absl::Status LocalAssetsStreamManagerServiceImpl::InitSsh(
+    const std::string& instance_id, const std::string& project_id,
+    const std::string& organization_id, std::string* instance_ip,
+    uint16_t* instance_port) {
+  SdkUtil sdk_util;
+  instance_ip->clear();
+  *instance_port = 0;
+
+  ProcessStartInfo start_info;
+  start_info.command = absl::StrFormat(
+      "%s ssh init", path::Join(sdk_util.GetDevBinPath(), "ggp"));
+  start_info.command += absl::StrFormat(" --instance %s", Quoted(instance_id));
+  if (!project_id.empty()) {
+    start_info.command += absl::StrFormat(" --project %s", Quoted(project_id));
+  }
+  if (!organization_id.empty()) {
+    start_info.command +=
+        absl::StrFormat(" --organization %s", Quoted(organization_id));
+  }
+  start_info.name = "ggp ssh init";
+
+  std::string output;
+  start_info.stdout_handler = [&output, this](const char* data,
+                                              size_t data_size) {
+    // Note: This is called from a background thread!
+    output.append(data, data_size);
+    return absl::OkStatus();
+  };
+  start_info.forward_output_to_log = true;
+
+  std::unique_ptr<Process> process = process_factory_->Create(start_info);
+  absl::Status status = process->Start();
+  if (!status.ok()) {
+    return WrapStatus(status, "Failed to start ggp process");
+  }
+
+  status = process->RunUntilExit();
+  if (!status.ok()) {
+    return WrapStatus(status, "Failed to run ggp process");
+  }
+
+  uint32_t exit_code = process->ExitCode();
+  if (exit_code != 0) {
+    return MakeStatus("ggp process exited with code %u", exit_code);
+  }
+
+  // Parse gamelet IP. Should be "Host: <instance_ip ip>".
+  if (!ParseValue(output, "Host", instance_ip)) {
+    return MakeStatus("Failed to parse host from ggp ssh init response\n%s",
+                      output);
+  }
+
+  // Parse ssh port. Should be "Port: <port>".
+  std::string port_string;
+  const bool result = ParseValue(output, "Port", &port_string);
+  int int_port = atoi(port_string.c_str());
+  if (!result || int_port == 0 || int_port <= 0 || int_port > UINT_MAX) {
+    return MakeStatus("Failed to parse ssh port from ggp ssh init response\n%s",
+                      output);
+  }
+
+  *instance_port = static_cast<uint16_t>(int_port);
+  return absl::OkStatus();
+}
+
+}  // namespace cdc_ft

asset_stream_manager/local_assets_stream_manager_service_impl.h

@@ -0,0 +1,90 @@
+/*
+ * Copyright 2022 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef ASSET_STREAM_MANAGER_LOCAL_ASSETS_STREAM_MANAGER_SERVICE_IMPL_H_
+#define ASSET_STREAM_MANAGER_LOCAL_ASSETS_STREAM_MANAGER_SERVICE_IMPL_H_
+
+#include "absl/status/status.h"
+#include "absl/status/statusor.h"
+#include "asset_stream_manager/session_config.h"
+#include "metrics/metrics.h"
+#include "proto/local_assets_stream_manager.grpc.pb.h"
+
+namespace cdc_ft {
+
+class MultiSession;
+class ProcessFactory;
+class SessionManager;
+
+// Implements a service to start and stop streaming sessions as a server.
+// The corresponding clients are implemented by the ggp CLI and SDK Proxy.
+// The CLI triggers StartSession() from `ggp instance mount --local-dir` and
+// StopSession() from `ggp instance unmount`. SDK Proxy invokes StartSession()
+// when a user starts a new game from the partner portal and sets an `Asset
+// streaming directory` in the `Advanced settings` in the `Play settings`
+// dialog.
+// This service is owned by SessionManagementServer.
+class LocalAssetsStreamManagerServiceImpl final
+    : public localassetsstreammanager::LocalAssetsStreamManager::Service {
+ public:
+  using StartSessionRequest = localassetsstreammanager::StartSessionRequest;
+  using StartSessionRequestOrigin =
+      localassetsstreammanager::StartSessionRequest_Origin;
+  using StartSessionResponse = localassetsstreammanager::StartSessionResponse;
+  using StopSessionRequest = localassetsstreammanager::StopSessionRequest;
+  using StopSessionResponse = localassetsstreammanager::StopSessionResponse;
+
+  LocalAssetsStreamManagerServiceImpl(
+      SessionManager* session_manager, ProcessFactory* process_factory,
+      metrics::MetricsService* const metrics_service);
+  ~LocalAssetsStreamManagerServiceImpl();
+
+  // Starts a streaming session from path |request->workstation_directory()| to
+  // the instance with id |request->gamelet_id()|. Stops an existing session
+  // if it exists.
+  grpc::Status StartSession(grpc::ServerContext* context,
+                            const StartSessionRequest* request,
+                            StartSessionResponse* response) override
+      ABSL_LOCKS_EXCLUDED(sessions_mutex_);
+
+  // Stops the streaming session to the instance with id
+  // |request->gamelet_id()|. Returns a NotFound error if no session exists.
+  grpc::Status StopSession(grpc::ServerContext* context,
+                           const StopSessionRequest* request,
+                           StopSessionResponse* response) override
+      ABSL_LOCKS_EXCLUDED(sessions_mutex_);
+
+ private:
+  // Convert StartSessionRequest enum to metrics enum.
+  metrics::RequestOrigin ConvertOrigin(StartSessionRequestOrigin origin) const;
+
+  // Initializes an ssh connection to a gamelet by calling 'ggp ssh init'.
+  // |instance_id| must be set, |project_id|, |organization_id| are optional.
+  // Returns |instance_ip| and |instance_port| (SSH port).
+  absl::Status InitSsh(const std::string& instance_id,
+                       const std::string& project_id,
+                       const std::string& organization_id,
+                       std::string* instance_ip, uint16_t* instance_port);
+
+  const SessionConfig cfg_;
+  SessionManager* const session_manager_;
+  ProcessFactory* const process_factory_;
+  metrics::MetricsService* const metrics_service_;
+};
+
+}  // namespace cdc_ft
+
+#endif  // ASSET_STREAM_MANAGER_LOCAL_ASSETS_STREAM_MANAGER_SERVICE_IMPL_H_

asset_stream_manager/main.cc

@@ -0,0 +1,182 @@
+// Copyright 2022 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "absl/flags/flag.h"
+#include "absl/flags/parse.h"
+#include "absl_helper/jedec_size_flag.h"
+#include "asset_stream_manager/asset_stream_config.h"
+#include "asset_stream_manager/background_service_impl.h"
+#include "asset_stream_manager/local_assets_stream_manager_service_impl.h"
+#include "asset_stream_manager/session_management_server.h"
+#include "asset_stream_manager/session_manager.h"
+#include "common/log.h"
+#include "common/path.h"
+#include "common/process.h"
+#include "common/sdk_util.h"
+#include "common/status_macros.h"
+#include "data_store/data_provider.h"
+#include "data_store/disk_data_store.h"
+#include "metrics/metrics.h"
+
+namespace cdc_ft {
+namespace {
+
+constexpr int kSessionManagementPort = 44432;
+
+absl::Status Run(const AssetStreamConfig& cfg) {
+  WinProcessFactory process_factory;
+  metrics::MetricsService metrics_service;
+
+  SessionManager session_manager(cfg.session_cfg(), &process_factory,
+                                 &metrics_service);
+  BackgroundServiceImpl background_service;
+  LocalAssetsStreamManagerServiceImpl session_service(
+      &session_manager, &process_factory, &metrics_service);
+
+  SessionManagementServer sm_server(&session_service, &background_service,
+                                    &session_manager);
+  background_service.SetExitCallback(
+      [&sm_server]() { return sm_server.Shutdown(); });
+
+  RETURN_IF_ERROR(sm_server.Start(kSessionManagementPort));
+  if (!cfg.src_dir().empty()) {
+    MultiSession* ms_unused;
+    metrics::SessionStartStatus status_unused;
+    RETURN_IF_ERROR(session_manager.StartSession(
+        /*instance_id=*/cfg.instance_ip(), /*project_id=*/std::string(),
+        /*organization_id=*/std::string(), cfg.instance_ip(),
+        cfg.instance_port(), cfg.src_dir(), &ms_unused, &status_unused));
+  }
+  sm_server.RunUntilShutdown();
+  return absl::OkStatus();
+}
+
+void InitLogging(bool log_to_stdout, int verbosity) {
+  LogLevel level = cdc_ft::Log::VerbosityToLogLevel(verbosity);
+  if (log_to_stdout) {
+    cdc_ft::Log::Initialize(std::make_unique<cdc_ft::ConsoleLog>(level));
+  } else {
+    SdkUtil util;
+    cdc_ft::Log::Initialize(std::make_unique<cdc_ft::FileLog>(
+        level, util.GetLogPath("assets_stream_manager_v3").c_str()));
+  }
+}
+
+// Declare AS20 flags, so that AS30 can be used on older SDKs simply by
+// replacing the binary. Note that the RETIRED_FLAGS macro can't be used
+// because the flags contain dashes. This code mimics the macro.
+absl::flags_internal::RetiredFlag<int> RETIRED_FLAGS_port;
+absl::flags_internal::RetiredFlag<std::string> RETIRED_FLAGS_session_ports;
+absl::flags_internal::RetiredFlag<std::string> RETIRED_FLAGS_gm_mount_point;
+absl::flags_internal::RetiredFlag<bool> RETIRED_FLAGS_allow_edge;
+const auto RETIRED_FLAGS_REG_port =
+    (RETIRED_FLAGS_port.Retire("port"),
+     ::absl::flags_internal::FlagRegistrarEmpty{});
+const auto RETIRED_FLAGS_REG_session_ports =
+    (RETIRED_FLAGS_session_ports.Retire("session-ports"),
+     ::absl::flags_internal::FlagRegistrarEmpty{});
+const auto RETIRED_FLAGS_REG_gm_mount_point =
+    (RETIRED_FLAGS_gm_mount_point.Retire("gamelet-mount-point"),
+     ::absl::flags_internal::FlagRegistrarEmpty{});
+const auto RETIRED_FLAGS_REG_allow_edge =
+    (RETIRED_FLAGS_allow_edge.Retire("allow-edge"),
+     ::absl::flags_internal::FlagRegistrarEmpty{});
+
+}  // namespace
+}  // namespace cdc_ft
+
+ABSL_FLAG(std::string, src_dir, "",
+          "Start a streaming session immediately from the given Windows path. "
+          "Used during development. Must have exactly one gamelet reserved or "
+          "specify the target gamelet with --instance.");
+ABSL_FLAG(std::string, instance_ip, "",
+          "Connect to the instance with the given IP address for this session. "
+          "This flag is ignored unless --src_dir is set as well. Used "
+          "during development. ");
+ABSL_FLAG(uint16_t, instance_port, 0,
+          "Connect to the instance through the given SSH port. "
+          "This flag is ignored unless --src_dir is set as well. Used "
+          "during development. ");
+ABSL_FLAG(int, verbosity, 2, "Verbosity of the log output");
+ABSL_FLAG(bool, debug, false, "Run FUSE filesystem in debug mode");
+ABSL_FLAG(bool, singlethreaded, false,
+          "Run FUSE filesystem in singlethreaded mode");
+ABSL_FLAG(bool, stats, false,
+          "Collect and print detailed streaming statistics");
+ABSL_FLAG(bool, quiet, false,
+          "Do not print any output except errors and stats");
+ABSL_FLAG(int, manifest_updater_threads, 4,
+          "Number of threads used to compute file hashes on the workstation.");
+ABSL_FLAG(int, file_change_wait_duration_ms, 500,
+          "Time in milliseconds to wait until pushing a file change to the "
+          "instance after detecting it.");
+ABSL_FLAG(bool, check, false, "Check FUSE consistency and log check results");
+ABSL_FLAG(bool, log_to_stdout, false, "Log to stdout instead of to a file");
+ABSL_FLAG(cdc_ft::JedecSize, cache_capacity,
+          cdc_ft::JedecSize(cdc_ft::DiskDataStore::kDefaultCapacity),
+          "Cache capacity. Supports common unit suffixes K, M, G.");
+ABSL_FLAG(uint32_t, cleanup_timeout, cdc_ft::DataProvider::kCleanupTimeoutSec,
+          "Period in seconds at which instance cache cleanups are run");
+ABSL_FLAG(uint32_t, access_idle_timeout, cdc_ft::DataProvider::kAccessIdleSec,
+          "Do not run instance cache cleanups for this many seconds after the "
+          "last file access");
+
+int main(int argc, char* argv[]) {
+  absl::ParseCommandLine(argc, argv);
+
+  // Set up config. Allow overriding this config with
+  // %APPDATA%\GGP\services\assets_stream_manager_v3.json.
+  cdc_ft::SdkUtil sdk_util;
+  const std::string config_path = cdc_ft::path::Join(
+      sdk_util.GetServicesConfigPath(), "assets_stream_manager_v3.json");
+  cdc_ft::AssetStreamConfig cfg;
+  absl::Status cfg_load_status = cfg.LoadFromFile(config_path);
+
+  cdc_ft::InitLogging(cfg.log_to_stdout(), cfg.session_cfg().verbosity);
+
+  // Log status of loaded configuration. Errors are not critical.
+  if (cfg_load_status.ok()) {
+    LOG_INFO("Successfully loaded configuration file at '%s'", config_path);
+  } else if (absl::IsNotFound(cfg_load_status)) {
+    LOG_INFO("No configuration file found at '%s'", config_path);
+  } else {
+    LOG_ERROR("%s", cfg_load_status.message());
+  }
+
+  std::string flags_read = cfg.GetFlagsReadFromFile();
+  if (!flags_read.empty()) {
+    LOG_INFO(
+        "The following settings were read from the configuration file and "
+        "override the corresponding command line flags if set: %s",
+        flags_read);
+  }
+
+  std::string flag_errors = cfg.GetFlagReadErrors();
+  if (!flag_errors.empty()) {
+    LOG_WARNING("%s", flag_errors);
+  }
+
+  LOG_DEBUG("Configuration:\n%s", cfg.ToString());
+
+  absl::Status status = cdc_ft::Run(cfg);
+  if (!status.ok()) {
+    LOG_ERROR("%s", status.ToString());
+  } else {
+    LOG_INFO("Asset stream manager shut down successfully.");
+  }
+
+  cdc_ft::Log::Shutdown();
+  static_assert(static_cast<int>(absl::StatusCode::kOk) == 0, "kOk not 0");
+  return static_cast<int>(status.code());
+}

asset_stream_manager/metrics_recorder.cc

@@ -0,0 +1,69 @@
+// Copyright 2022 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "asset_stream_manager/metrics_recorder.h"
+
+#include "common/log.h"
+
+namespace cdc_ft {
+
+MetricsRecorder::MetricsRecorder(metrics::MetricsService* const metrics_service)
+    : metrics_service_(metrics_service) {}
+
+metrics::MetricsService* MetricsRecorder::GetMetricsService() const {
+  return metrics_service_;
+}
+
+MultiSessionMetricsRecorder::MultiSessionMetricsRecorder(
+    metrics::MetricsService* const metrics_service)
+    : MetricsRecorder(metrics_service),
+      multisession_id_(Util::GenerateUniqueId()) {}
+
+MultiSessionMetricsRecorder::~MultiSessionMetricsRecorder() = default;
+
+void MultiSessionMetricsRecorder::RecordEvent(metrics::DeveloperLogEvent event,
+                                              metrics::EventType code) const {
+  if (!event.as_manager_data) {
+    event.as_manager_data =
+        std::make_unique<metrics::AssetStreamingManagerData>();
+  }
+  event.as_manager_data->multisession_id = multisession_id_;
+  metrics_service_->RecordEvent(std::move(event), code);
+}
+
+SessionMetricsRecorder::SessionMetricsRecorder(
+    metrics::MetricsService* const metrics_service,
+    const std::string& multisession_id, const std::string& project_id,
+    const std::string& organization_id)
+    : MetricsRecorder(metrics_service),
+      multisession_id_(multisession_id),
+      project_id_(project_id),
+      organization_id_(organization_id),
+      session_id_(Util::GenerateUniqueId()) {}
+
+SessionMetricsRecorder::~SessionMetricsRecorder() = default;
+
+void SessionMetricsRecorder::RecordEvent(metrics::DeveloperLogEvent event,
+                                         metrics::EventType code) const {
+  if (!event.as_manager_data) {
+    event.as_manager_data =
+        std::make_unique<metrics::AssetStreamingManagerData>();
+  }
+  event.as_manager_data->multisession_id = multisession_id_;
+  event.as_manager_data->session_id = session_id_;
+  event.project_id = project_id_;
+  event.organization_id = organization_id_;
+  metrics_service_->RecordEvent(std::move(event), code);
+}
+}  // namespace cdc_ft

asset_stream_manager/metrics_recorder.h

@@ -0,0 +1,77 @@
+/*
+ * Copyright 2022 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef ASSET_STREAM_MANAGER_METRICS_RECORDER_H_
+#define ASSET_STREAM_MANAGER_METRICS_RECORDER_H_
+
+#include "absl/status/status.h"
+#include "common/util.h"
+#include "metrics/enums.h"
+#include "metrics/messages.h"
+#include "metrics/metrics.h"
+
+namespace cdc_ft {
+
+class MetricsRecorder {
+ public:
+  virtual void RecordEvent(metrics::DeveloperLogEvent event,
+                           metrics::EventType code) const = 0;
+
+  virtual metrics::MetricsService* GetMetricsService() const;
+
+ protected:
+  explicit MetricsRecorder(metrics::MetricsService* const metrics_service);
+  metrics::MetricsService* const metrics_service_;
+};
+
+class MultiSessionMetricsRecorder : public MetricsRecorder {
+ public:
+  explicit MultiSessionMetricsRecorder(
+      metrics::MetricsService* const metrics_service);
+  ~MultiSessionMetricsRecorder();
+
+  virtual void RecordEvent(metrics::DeveloperLogEvent event,
+                           metrics::EventType code) const;
+
+  const std::string& MultiSessionId() const { return multisession_id_; }
+
+ private:
+  std::string multisession_id_;
+};
+
+class SessionMetricsRecorder : public MetricsRecorder {
+ public:
+  explicit SessionMetricsRecorder(
+      metrics::MetricsService* const metrics_service,
+      const std::string& multisession_id, const std::string& project_id,
+      const std::string& organization_id);
+  ~SessionMetricsRecorder();
+
+  virtual void RecordEvent(metrics::DeveloperLogEvent event,
+                           metrics::EventType code) const;
+
+  const std::string& SessionId() const { return session_id_; }
+
+ private:
+  std::string multisession_id_;
+  std::string session_id_;
+  std::string project_id_;
+  std::string organization_id_;
+};
+
+}  // namespace cdc_ft
+
+#endif  // ASSET_STREAM_MANAGER_METRICS_RECORDER_H_

asset_stream_manager/metrics_recorder_test.cc

@@ -0,0 +1,131 @@
+// Copyright 2022 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "asset_stream_manager/metrics_recorder.h"
+
+#include "common/status_test_macros.h"
+#include "gtest/gtest.h"
+#include "metrics/metrics.h"
+
+namespace cdc_ft {
+namespace {
+
+struct MetricsRecord {
+  MetricsRecord(metrics::DeveloperLogEvent dev_log_event,
+                metrics::EventType code)
+      : dev_log_event(std::move(dev_log_event)), code(code) {}
+  metrics::DeveloperLogEvent dev_log_event;
+  metrics::EventType code;
+};
+
+class MetricsServiceForTesting : public metrics::MetricsService {
+ public:
+  MetricsServiceForTesting() {
+    metrics_records_ = new std::vector<MetricsRecord>();
+  }
+
+  ~MetricsServiceForTesting() { delete metrics_records_; }
+
+  void RecordEvent(metrics::DeveloperLogEvent event,
+                   metrics::EventType code) const override {
+    metrics_records_->push_back(MetricsRecord(std::move(event), code));
+  }
+
+  int NumberOfRecordRequests() { return (int)metrics_records_->size(); }
+
+  std::vector<MetricsRecord> GetEventsAndClear() {
+    return std::move(*metrics_records_);
+  }
+
+ private:
+  std::vector<MetricsRecord>* metrics_records_;
+};
+
+class MetricsRecorderTest : public ::testing::Test {
+ public:
+  void SetUp() override { metrics_service_ = new MetricsServiceForTesting(); }
+
+  void TearDown() override { delete metrics_service_; }
+
+ protected:
+  MetricsServiceForTesting* metrics_service_;
+};
+
+TEST_F(MetricsRecorderTest, SendEventWithMultisessionId) {
+  MultiSessionMetricsRecorder target(metrics_service_);
+  metrics::DeveloperLogEvent q_evt;
+  q_evt.project_id = "proj/id";
+  q_evt.organization_id = "org/id";
+
+  target.RecordEvent(std::move(q_evt), metrics::EventType::kMultiSessionStart);
+  EXPECT_EQ(metrics_service_->NumberOfRecordRequests(), 1);
+  std::vector<MetricsRecord> requests = metrics_service_->GetEventsAndClear();
+  EXPECT_EQ(requests[0].code, metrics::EventType::kMultiSessionStart);
+  metrics::DeveloperLogEvent expected_evt;
+  expected_evt.project_id = "proj/id";
+  expected_evt.organization_id = "org/id";
+  expected_evt.as_manager_data =
+      std::make_unique<metrics::AssetStreamingManagerData>();
+  expected_evt.as_manager_data->multisession_id = target.MultiSessionId();
+  EXPECT_EQ(requests[0].dev_log_event, expected_evt);
+  EXPECT_FALSE(target.MultiSessionId().empty());
+
+  q_evt = metrics::DeveloperLogEvent();
+  q_evt.project_id = "proj/id";
+  q_evt.organization_id = "org/id";
+  target.RecordEvent(std::move(q_evt), metrics::EventType::kMultiSessionStart);
+  EXPECT_EQ(metrics_service_->NumberOfRecordRequests(), 1);
+  std::vector<MetricsRecord> requests2 = metrics_service_->GetEventsAndClear();
+  EXPECT_EQ(requests2[0].code, metrics::EventType::kMultiSessionStart);
+  EXPECT_EQ(requests2[0].dev_log_event, requests[0].dev_log_event);
+
+  MultiSessionMetricsRecorder target2(metrics_service_);
+  EXPECT_NE(target2.MultiSessionId(), target.MultiSessionId());
+}
+
+TEST_F(MetricsRecorderTest, SendEventWithSessionId) {
+  SessionMetricsRecorder target(metrics_service_, "id1", "m_proj", "m_org");
+  metrics::DeveloperLogEvent q_evt;
+  q_evt.project_id = "proj/id";
+  q_evt.organization_id = "org/id";
+
+  target.RecordEvent(std::move(q_evt), metrics::EventType::kSessionStart);
+  EXPECT_EQ(metrics_service_->NumberOfRecordRequests(), 1);
+  std::vector<MetricsRecord> requests = metrics_service_->GetEventsAndClear();
+  EXPECT_EQ(requests[0].code, metrics::EventType::kSessionStart);
+  metrics::DeveloperLogEvent expected_evt;
+  expected_evt.project_id = "m_proj";
+  expected_evt.organization_id = "m_org";
+  expected_evt.as_manager_data =
+      std::make_unique<metrics::AssetStreamingManagerData>();
+  expected_evt.as_manager_data->multisession_id = "id1";
+  expected_evt.as_manager_data->session_id = target.SessionId();
+  EXPECT_EQ(requests[0].dev_log_event, expected_evt);
+  EXPECT_FALSE(target.SessionId().empty());
+
+  q_evt = metrics::DeveloperLogEvent();
+  q_evt.project_id = "proj/id";
+  q_evt.organization_id = "org/id";
+  target.RecordEvent(std::move(q_evt), metrics::EventType::kSessionStart);
+  EXPECT_EQ(metrics_service_->NumberOfRecordRequests(), 1);
+  std::vector<MetricsRecord> requests2 = metrics_service_->GetEventsAndClear();
+  EXPECT_EQ(requests2[0].code, metrics::EventType::kSessionStart);
+  EXPECT_EQ(requests2[0].dev_log_event, requests[0].dev_log_event);
+
+  SessionMetricsRecorder target2(metrics_service_, "id2", "m_proj", "m_org");
+  EXPECT_NE(target2.SessionId(), target.SessionId());
+}
+
+}  // namespace
+}  // namespace cdc_ft

asset_stream_manager/multi_session.cc

@@ -0,0 +1,699 @@
+// Copyright 2022 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "asset_stream_manager/multi_session.h"
+
+#include "asset_stream_manager/session.h"
+#include "common/file_watcher_win.h"
+#include "common/log.h"
+#include "common/path.h"
+#include "common/platform.h"
+#include "common/port_manager.h"
+#include "common/process.h"
+#include "common/util.h"
+#include "data_store/disk_data_store.h"
+#include "manifest/content_id.h"
+#include "manifest/manifest_iterator.h"
+#include "manifest/manifest_printer.h"
+#include "manifest/manifest_proto_defs.h"
+#include "metrics/enums.h"
+#include "metrics/messages.h"
+
+namespace cdc_ft {
+namespace {
+
+// Ports used by the asset streaming service for local port forwarding on
+// workstation and gamelet.
+constexpr int kAssetStreamPortFirst = 44433;
+constexpr int kAssetStreamPortLast = 44442;
+
+// Stats output period (if enabled).
+constexpr double kStatsPrintDelaySec = 0.1f;
+
+ManifestUpdater::Operator FileWatcherActionToOperation(
+    FileWatcherWin::FileAction action) {
+  switch (action) {
+    case FileWatcherWin::FileAction::kAdded:
+      return ManifestUpdater::Operator::kAdd;
+    case FileWatcherWin::FileAction::kModified:
+      return ManifestUpdater::Operator::kUpdate;
+    case FileWatcherWin::FileAction::kDeleted:
+      return ManifestUpdater::Operator::kDelete;
+  }
+  // The switch must cover all actions.
+  LOG_ERROR("Unhandled action: %d", static_cast<int>(action));
+  assert(false);
+  return ManifestUpdater::Operator::kAdd;
+}
+
+// Converts |modified_files| (as returned from the file watcher) into an
+// OperationList (as required by the manifest updater).
+ManifestUpdater::OperationList GetFileOperations(
+    const FileWatcherWin::FileMap& modified_files) {
+  AssetInfo ai;
+  ManifestUpdater::OperationList ops;
+  ops.reserve(modified_files.size());
+  for (const auto& [path, info] : modified_files) {
+    ai.path = path;
+    ai.type = info.is_dir ? AssetProto::DIRECTORY : AssetProto::FILE;
+    ai.size = info.size;
+    ai.mtime = info.mtime;
+    ops.emplace_back(FileWatcherActionToOperation(info.action), std::move(ai));
+  }
+  return ops;
+}
+
+}  // namespace
+
+MultiSessionRunner::MultiSessionRunner(
+    std::string src_dir, DataStoreWriter* data_store,
+    ProcessFactory* process_factory, bool enable_stats,
+    absl::Duration wait_duration, uint32_t num_updater_threads,
+    MultiSessionMetricsRecorder const* metrics_recorder,
+    ManifestUpdatedCb manifest_updated_cb)
+    : src_dir_(std::move(src_dir)),
+      data_store_(data_store),
+      process_factory_(process_factory),
+      file_chunks_(enable_stats),
+      wait_duration_(wait_duration),
+      num_updater_threads_(num_updater_threads),
+      manifest_updated_cb_(std::move(manifest_updated_cb)),
+      metrics_recorder_(metrics_recorder) {
+  assert(metrics_recorder_);
+}
+
+absl::Status MultiSessionRunner::Initialize(int port,
+                                            AssetStreamServerType type,
+                                            ContentSentHandler content_sent) {
+  // Start the server.
+  assert(!server_);
+  server_ = AssetStreamServer::Create(type, src_dir_, data_store_,
+                                      &file_chunks_, content_sent);
+  assert(server_);
+  RETURN_IF_ERROR(server_->Start(port),
+                  "Failed to start asset stream server for '%s'", src_dir_);
+
+  assert(!thread_);
+  thread_ = std::make_unique<std::thread>([this]() { Run(); });
+
+  return absl::OkStatus();
+}
+
+absl::Status MultiSessionRunner::Shutdown() {
+  // Send shutdown signal.
+  {
+    absl::MutexLock lock(&mutex_);
+    shutdown_ = true;
+  }
+  if (thread_) {
+    if (thread_->joinable()) thread_->join();
+    thread_.reset();
+  }
+
+  // Shut down asset stream server.
+  if (server_) {
+    server_->Shutdown();
+    server_.reset();
+  }
+
+  return status_;
+}
+
+absl::Status MultiSessionRunner::WaitForManifestAck(
+    const std::string& instance_id, absl::Duration fuse_timeout) {
+  {
+    absl::MutexLock lock(&mutex_);
+
+    LOG_INFO("Waiting for manifest to be available");
+    auto cond = [this]() { return manifest_set_ || !status_.ok(); };
+    mutex_.Await(absl::Condition(&cond));
+
+    if (!status_.ok())
+      return WrapStatus(status_, "Failed to set up streaming session for '%s'",
+                        src_dir_);
+  }
+
+  LOG_INFO("Waiting for FUSE ack");
+  assert(server_);
+  RETURN_IF_ERROR(server_->WaitForManifestAck(instance_id, fuse_timeout));
+
+  return absl::OkStatus();
+}
+
+absl::Status MultiSessionRunner::Status() {
+  absl::MutexLock lock(&mutex_);
+  return status_;
+}
+
+ContentIdProto MultiSessionRunner::ManifestId() const {
+  assert(server_);
+  return server_->GetManifestId();
+}
+
+void MultiSessionRunner::Run() {
+  // Create the manifest updater.
+  UpdaterConfig cfg;
+  cfg.num_threads = num_updater_threads_;
+  cfg.src_dir = src_dir_;
+  ManifestUpdater manifest_updater(data_store_, std::move(cfg));
+
+  // Set up file watcher.
+  // The streamed path should be a directory and exist at the beginning.
+  FileWatcherWin watcher(src_dir_);
+  absl::Status status = watcher.StartWatching([this]() { OnFilesChanged(); },
+                                              [this]() { OnDirRecreated(); });
+  if (!status.ok()) {
+    SetStatus(
+        WrapStatus(status, "Failed to update manifest for '%s'", src_dir_));
+    return;
+  }
+
+  // Push an intermediate manifest containing the full directory structure, but
+  // potentially missing chunks. The purpose is that the FUSE can immediately
+  // show the structure and inode stats. FUSE will block on file reads that
+  // cannot be served due to missing chunks until the manifest is ready.
+  auto push_intermediate_manifest = [this](const ContentIdProto& manifest_id) {
+    SetManifest(manifest_id);
+  };
+
+  // Bring the manifest up to date.
+  LOG_INFO("Updating manifest for '%s'...", src_dir_);
+  Stopwatch sw;
+  status =
+      manifest_updater.UpdateAll(&file_chunks_, push_intermediate_manifest);
+  RecordManifestUpdate(manifest_updater, sw.Elapsed(),
+                       metrics::UpdateTrigger::kInitUpdateAll, status);
+  if (!status.ok()) {
+    SetStatus(
+        WrapStatus(status, "Failed to update manifest for '%s'", src_dir_));
+    return;
+  }
+  RecordMultiSessionStart(manifest_updater);
+  SetManifest(manifest_updater.ManifestId());
+  LOG_INFO("Manifest for '%s' updated in %0.3f seconds", src_dir_,
+           sw.ElapsedSeconds());
+
+  while (!shutdown_) {
+    FileWatcherWin::FileMap modified_files;
+    bool clean_manifest = false;
+    {
+      // Wait for changes.
+      absl::MutexLock lock(&mutex_);
+
+      bool prev_files_changed = files_changed_;
+      absl::Duration timeout =
+          absl::Seconds(file_chunks_.HasStats() ? kStatsPrintDelaySec : 3600.0);
+      if (files_changed_) {
+        timeout = std::max(wait_duration_ - files_changed_timer_.Elapsed(),
+                           absl::Milliseconds(1));
+      } else {
+        files_changed_timer_.Reset();
+      }
+      auto cond = [this]() {
+        return shutdown_ || files_changed_ || dir_recreated_;
+      };
+      mutex_.AwaitWithTimeout(absl::Condition(&cond), timeout);
+
+      // If |files_changed_| became true, wait some more time before updating
+      // the manifest.
+      if (!prev_files_changed && files_changed_) files_changed_timer_.Reset();
+
+      // Shut down.
+      if (shutdown_) {
+        LOG_INFO("MultiSession('%s'): Shutting down", src_dir_);
+        break;
+      }
+
+      // Pick up modified files.
+      if (!dir_recreated_ && files_changed_ &&
+          files_changed_timer_.Elapsed() > wait_duration_) {
+        modified_files = watcher.GetModifiedFiles();
+        files_changed_ = false;
+        files_changed_timer_.Reset();
+      }
+
+      if (dir_recreated_) {
+        clean_manifest = true;
+        dir_recreated_ = false;
+      }
+    }  // mutex_ lock
+
+    if (clean_manifest) {
+      LOG_DEBUG(
+          "Streamed directory '%s' was possibly re-created or not all changes "
+          "were detected, re-building the manifest",
+          src_dir_);
+      modified_files.clear();
+      sw.Reset();
+      status = manifest_updater.UpdateAll(&file_chunks_);
+      RecordManifestUpdate(manifest_updater, sw.Elapsed(),
+                           metrics::UpdateTrigger::kRunningUpdateAll, status);
+      if (!status.ok()) {
+        LOG_WARNING(
+            "Updating manifest for '%s' after re-creating directory failed: "
+            "'%s'",
+            src_dir_, status.ToString());
+        SetManifest(manifest_updater.DefaultManifestId());
+      } else {
+        SetManifest(manifest_updater.ManifestId());
+      }
+    } else if (!modified_files.empty()) {
+      ManifestUpdater::OperationList ops = GetFileOperations(modified_files);
+      sw.Reset();
+      status = manifest_updater.Update(&ops, &file_chunks_);
+      RecordManifestUpdate(manifest_updater, sw.Elapsed(),
+                           metrics::UpdateTrigger::kRegularUpdate, status);
+      if (!status.ok()) {
+        LOG_WARNING("Updating manifest for '%s' failed: %s", src_dir_,
+                    status.ToString());
+        SetManifest(manifest_updater.DefaultManifestId());
+      } else {
+        SetManifest(manifest_updater.ManifestId());
+      }
+    }
+
+    // Update stats output.
+    file_chunks_.PrintStats();
+  }
+}
+
+void MultiSessionRunner::RecordManifestUpdate(
+    const ManifestUpdater& manifest_updater, absl::Duration duration,
+    metrics::UpdateTrigger trigger, absl::Status status) {
+  metrics::DeveloperLogEvent evt;
+  evt.as_manager_data = std::make_unique<metrics::AssetStreamingManagerData>();
+  evt.as_manager_data->manifest_update_data =
+      std::make_unique<metrics::ManifestUpdateData>();
+  evt.as_manager_data->manifest_update_data->local_duration_ms =
+      absl::ToInt64Milliseconds(duration);
+  evt.as_manager_data->manifest_update_data->status = status.code();
+  evt.as_manager_data->manifest_update_data->trigger = trigger;
+  const UpdaterStats& stats = manifest_updater.Stats();
+  evt.as_manager_data->manifest_update_data->total_assets_added_or_updated =
+      stats.total_assets_added_or_updated;
+  evt.as_manager_data->manifest_update_data->total_assets_deleted =
+      stats.total_assets_deleted;
+  evt.as_manager_data->manifest_update_data->total_chunks = stats.total_chunks;
+  evt.as_manager_data->manifest_update_data->total_files_added_or_updated =
+      stats.total_files_added_or_updated;
+  evt.as_manager_data->manifest_update_data->total_files_failed =
+      stats.total_files_failed;
+  evt.as_manager_data->manifest_update_data->total_processed_bytes =
+      stats.total_processed_bytes;
+  metrics_recorder_->RecordEvent(std::move(evt),
+                                 metrics::EventType::kManifestUpdated);
+}
+
+void MultiSessionRunner::RecordMultiSessionStart(
+    const ManifestUpdater& manifest_updater) {
+  metrics::DeveloperLogEvent evt;
+  evt.as_manager_data = std::make_unique<metrics::AssetStreamingManagerData>();
+  evt.as_manager_data->multi_session_start_data =
+      std::make_unique<metrics::MultiSessionStartData>();
+  ManifestIterator manifest_iter(data_store_);
+  absl::Status status = manifest_iter.Open(manifest_updater.ManifestId());
+  if (status.ok()) {
+    const AssetProto* entry = &manifest_iter.Manifest().root_dir();
+    uint32_t file_count = 0;
+    uint64_t total_chunks = 0;
+    uint64_t total_processed_bytes = 0;
+    do {
+      if (entry->type() == AssetProto::FILE) {
+        ++file_count;
+        total_chunks += entry->file_chunks_size();
+        total_processed_bytes += entry->file_size();
+        for (const IndirectChunkListProto& icl :
+             entry->file_indirect_chunks()) {
+          ChunkListProto list;
+          status = data_store_->GetProto(icl.chunk_list_id(), &list);
+          if (status.ok()) {
+            total_chunks += list.chunks_size();
+          } else {
+            LOG_WARNING("Could not get proto by id: '%s'. %s",
+                        ContentId::ToHexString(icl.chunk_list_id()),
+                        status.ToString());
+          }
+        }
+      }
+    } while ((entry = manifest_iter.NextEntry()) != nullptr);
+    evt.as_manager_data->multi_session_start_data->file_count = file_count;
+    evt.as_manager_data->multi_session_start_data->chunk_count = total_chunks;
+    evt.as_manager_data->multi_session_start_data->byte_count =
+        total_processed_bytes;
+  } else {
+    LOG_WARNING("Could not open manifest by id: '%s'. %s",
+                ContentId::ToHexString(manifest_updater.ManifestId()),
+                status.ToString());
+  }
+  evt.as_manager_data->multi_session_start_data->min_chunk_size =
+      static_cast<uint64_t>(manifest_updater.Config().min_chunk_size);
+  evt.as_manager_data->multi_session_start_data->avg_chunk_size =
+      static_cast<uint64_t>(manifest_updater.Config().avg_chunk_size);
+  evt.as_manager_data->multi_session_start_data->max_chunk_size =
+      static_cast<uint64_t>(manifest_updater.Config().max_chunk_size);
+  metrics_recorder_->RecordEvent(std::move(evt),
+                                 metrics::EventType::kMultiSessionStart);
+}
+
+void MultiSessionRunner::SetStatus(absl::Status status)
+    ABSL_LOCKS_EXCLUDED(mutex_) {
+  absl::MutexLock lock(&mutex_);
+  status_ = std::move(status);
+}
+
+void MultiSessionRunner::OnFilesChanged() {
+  absl::MutexLock lock(&mutex_);
+  files_changed_ = true;
+}
+
+void MultiSessionRunner::OnDirRecreated() {
+  absl::MutexLock lock(&mutex_);
+  dir_recreated_ = true;
+}
+
+void MultiSessionRunner::SetManifest(const ContentIdProto& manifest_id) {
+  server_->SetManifestId(manifest_id);
+  if (Log::Instance()->GetLogLevel() <= LogLevel::kVerbose) {
+    ManifestPrinter printer;
+    ManifestProto manifest_proto;
+    absl::Status status = data_store_->GetProto(manifest_id, &manifest_proto);
+    std::string manifest_text;
+    printer.PrintToString(manifest_proto, &manifest_text);
+    if (status.ok()) {
+      LOG_DEBUG("Set manifest '%s'\n'%s'", ContentId::ToHexString(manifest_id),
+                manifest_text);
+    } else {
+      LOG_WARNING("Could not retrieve manifest from the data store '%s'",
+                  ContentId::ToHexString(manifest_id));
+    }
+  }
+
+  // Notify thread that starts the streaming session that a manifest has been
+  // set.
+  absl::MutexLock lock(&mutex_);
+  manifest_set_ = true;
+  if (manifest_updated_cb_) {
+    manifest_updated_cb_();
+  }
+}
+
+MultiSession::MultiSession(std::string src_dir, SessionConfig cfg,
+                           ProcessFactory* process_factory,
+                           MultiSessionMetricsRecorder const* metrics_recorder,
+                           std::unique_ptr<DataStoreWriter> data_store)
+    : src_dir_(src_dir),
+      cfg_(cfg),
+      process_factory_(process_factory),
+      data_store_(std::move(data_store)),
+      metrics_recorder_(metrics_recorder) {
+  assert(metrics_recorder_);
+}
+
+MultiSession::~MultiSession() {
+  absl::Status status = Shutdown();
+  if (!status.ok()) {
+    LOG_ERROR("Shutdown streaming from '%s' failed: %s", src_dir_,
+              status.ToString());
+  }
+}
+
+absl::Status MultiSession::Initialize() {
+  // |data_store_| is not set in production, but it can be overridden for tests.
+  if (!data_store_) {
+    std::string cache_path;
+    ASSIGN_OR_RETURN(cache_path, GetCachePath(src_dir_));
+    ASSIGN_OR_RETURN(data_store_,
+                     DiskDataStore::Create(/*depth=*/0, cache_path,
+                                           /*create_dirs=*/true),
+                     "Failed to create data store for '%s'", cache_path);
+  }
+
+  // Find an available local port.
+  std::unordered_set<int> ports;
+  ASSIGN_OR_RETURN(
+      ports,
+      PortManager::FindAvailableLocalPorts(kAssetStreamPortFirst,
+                                           kAssetStreamPortLast, "127.0.0.1",
+                                           process_factory_, true),
+      "Failed to find an available local port in the range [%d, %d]",
+      kAssetStreamPortFirst, kAssetStreamPortLast);
+  assert(!ports.empty());
+  local_asset_stream_port_ = *ports.begin();
+
+  assert(!runner_);
+  runner_ = std::make_unique<MultiSessionRunner>(
+      src_dir_, data_store_.get(), process_factory_, cfg_.stats,
+      absl::Milliseconds(cfg_.file_change_wait_duration_ms),
+      cfg_.manifest_updater_threads, metrics_recorder_);
+  RETURN_IF_ERROR(runner_->Initialize(
+                      local_asset_stream_port_, AssetStreamServerType::kGrpc,
+                      [this](uint64_t bc, uint64_t cc, std::string id) {
+                        this->OnContentSent(bc, cc, id);
+                      }),
+                  "Failed to initialize session runner");
+  StartHeartBeatCheck();
+  return absl::OkStatus();
+}
+
+absl::Status MultiSession::Shutdown() {
+  // Stop all sessions.
+  // TODO: Record error on multi-session end.
+  metrics_recorder_->RecordEvent(metrics::DeveloperLogEvent(),
+                                 metrics::EventType::kMultiSessionEnd);
+  {
+    absl::WriterMutexLock lock(&shutdownMu_);
+    shutdown_ = true;
+  }
+  while (!sessions_.empty()) {
+    std::string instance_id = sessions_.begin()->first;
+    RETURN_IF_ERROR(StopSession(instance_id),
+                    "Failed to stop session for instance id %s", instance_id);
+    sessions_.erase(instance_id);
+  }
+
+  if (runner_) {
+    RETURN_IF_ERROR(runner_->Shutdown());
+  }
+
+  if (heartbeat_watcher_.joinable()) {
+    heartbeat_watcher_.join();
+  }
+
+  return absl::OkStatus();
+}
+
+absl::Status MultiSession::Status() {
+  return runner_ ? runner_->Status() : absl::OkStatus();
+}
+
+absl::Status MultiSession::StartSession(const std::string& instance_id,
+                                        const std::string& project_id,
+                                        const std::string& organization_id,
+                                        const std::string& instance_ip,
+                                        uint16_t instance_port) {
+  absl::MutexLock lock(&sessions_mutex_);
+
+  if (sessions_.find(instance_id) != sessions_.end()) {
+    return absl::InvalidArgumentError(absl::StrFormat(
+        "Session for instance id '%s' already exists", instance_id));
+  }
+
+  if (!runner_)
+    return absl::FailedPreconditionError("MultiSession not started");
+
+  absl::Status runner_status = runner_->Status();
+  if (!runner_status.ok()) {
+    return WrapStatus(runner_status,
+                      "Failed to set up streaming session for '%s'", src_dir_);
+  }
+
+  auto metrics_recorder = std::make_unique<SessionMetricsRecorder>(
+      metrics_recorder_->GetMetricsService(),
+      metrics_recorder_->MultiSessionId(), project_id, organization_id);
+
+  auto session =
+      std::make_unique<Session>(instance_id, instance_ip, instance_port, cfg_,
+                                process_factory_, std::move(metrics_recorder));
+  RETURN_IF_ERROR(session->Start(local_asset_stream_port_,
+                                 kAssetStreamPortFirst, kAssetStreamPortLast));
+
+  // Wait for the FUSE to receive the intermediate manifest.
+  RETURN_IF_ERROR(runner_->WaitForManifestAck(instance_id, absl::Seconds(5)));
+
+  sessions_[instance_id] = std::move(session);
+  return absl::OkStatus();
+}
+
+absl::Status MultiSession::StopSession(const std::string& instance_id) {
+  absl::MutexLock lock(&sessions_mutex_);
+
+  if (sessions_.find(instance_id) == sessions_.end()) {
+    return absl::NotFoundError(
+        absl::StrFormat("No session for instance id '%s' found", instance_id));
+  }
+
+  if (!runner_)
+    return absl::FailedPreconditionError("MultiSession not started");
+
+  RETURN_IF_ERROR(sessions_[instance_id]->Stop());
+  sessions_.erase(instance_id);
+  return absl::OkStatus();
+}
+
+bool MultiSession::HasSessionForInstance(const std::string& instance_id) {
+  absl::ReaderMutexLock lock(&sessions_mutex_);
+  return sessions_.find(instance_id) != sessions_.end();
+}
+
+bool MultiSession::IsSessionHealthy(const std::string& instance_id) {
+  absl::ReaderMutexLock lock(&sessions_mutex_);
+  auto iter = sessions_.find(instance_id);
+  if (iter == sessions_.end()) return false;
+  Session* session = iter->second.get();
+  assert(session);
+  return session->IsHealthy();
+}
+
+bool MultiSession::Empty() {
+  absl::ReaderMutexLock lock(&sessions_mutex_);
+  return sessions_.empty();
+}
+
+uint32_t MultiSession::GetSessionCount() {
+  absl::ReaderMutexLock lock(&sessions_mutex_);
+  return static_cast<uint32_t>(sessions_.size());
+}
+
+// static
+std::string MultiSession::GetCacheDir(std::string dir) {
+  // Get full path, or else "..\foo" and "C:\foo" are treated differently, even
+  // if they map to the same directory.
+  dir = path::GetFullPath(dir);
+#if PLATFORM_WINDOWS
+  // On Windows, casing is ignored.
+  std::for_each(dir.begin(), dir.end(), [](char& c) { c = tolower(c); });
+#endif
+  path::EnsureEndsWithPathSeparator(&dir);
+  dir = path::ToUnix(std::move(dir));
+  ContentIdProto id = ContentId::FromDataString(dir);
+
+  // Replace invalid characters by _.
+  std::for_each(dir.begin(), dir.end(), [](char& c) {
+    static constexpr char invalid_chars[] = "<>:\"/\\|?*";
+    if (strchr(invalid_chars, c)) c = '_';
+  });
+
+  return dir + ContentId::ToHexString(id).substr(0, kDirHashLen);
+}
+
+// static
+absl::StatusOr<std::string> MultiSession::GetCachePath(
+    const std::string& src_dir, size_t max_len) {
+  std::string appdata_path;
+#if PLATFORM_WINDOWS
+  RETURN_IF_ERROR(
+      path::GetKnownFolderPath(path::FolderId::kRoamingAppData, &appdata_path),
+      "Failed to get roaming appdata path");
+#elif PLATFORM_LINUX
+  RETURN_IF_ERROR(path::GetEnv("HOME", &appdata_path));
+  path::Append(&appdata_path, ".cache");
+#endif
+
+  std::string base_dir = path::Join(appdata_path, "GGP", "asset_streaming");
+  std::string cache_dir = GetCacheDir(src_dir);
+
+  size_t total_size = base_dir.size() + 1 + cache_dir.size();
+  if (total_size <= max_len) return path::Join(base_dir, cache_dir);
+
+  // Path needs to be shortened. Remove |to_remove| many chars from the
+  // beginning of |cache_dir|, but keep the hash (last kDirHashLen bytes).
+  size_t to_remove = total_size - max_len;
+  assert(cache_dir.size() >= kDirHashLen);
+  if (to_remove > cache_dir.size() - kDirHashLen)
+    to_remove = cache_dir.size() - kDirHashLen;
+
+  // Remove UTF8 code points from the beginning.
+  size_t start = 0;
+  while (start < to_remove) {
+    int codepoint_len = Util::Utf8CodePointLen(cache_dir.data() + start);
+    // For invalid code points (codepoint_len == 0), just eat byte by byte.
+    start += std::max(codepoint_len, 1);
+  }
+
+  assert(start + kDirHashLen <= cache_dir.size());
+  return path::Join(base_dir, cache_dir.substr(start));
+}
+
+void MultiSession::RecordMultiSessionEvent(metrics::DeveloperLogEvent event,
+                                           metrics::EventType code) {
+  metrics_recorder_->RecordEvent(std::move(event), code);
+}
+
+void MultiSession::RecordSessionEvent(metrics::DeveloperLogEvent event,
+                                      metrics::EventType code,
+                                      const std::string& instance_id) {
+  Session* session = FindSession(instance_id);
+  if (session) {
+    session->RecordEvent(std::move(event), code);
+  }
+}
+
+Session* MultiSession::FindSession(const std::string& instance_id) {
+  absl::ReaderMutexLock lock(&sessions_mutex_);
+  auto session_it = sessions_.find(instance_id);
+  if (session_it == sessions_.end()) {
+    return nullptr;
+  }
+  return session_it->second.get();
+}
+
+void MultiSession::OnContentSent(size_t byte_count, size_t chunck_count,
+                                 std::string instance_id) {
+  if (instance_id.empty()) {
+    // |instance_id| is empty only in case when manifest wasn't acknowledged by
+    // the gamelet yet (ConfigStreamServiceImpl::AckManifestIdReceived was not
+    // invoked). This means MultiSession::StartSession is still waiting for
+    // manifest acknowledge and |sessions_mutex_| is currently locked. In this
+    // case invoking MultiSession::FindSession and waiting for |sessions_mutex_|
+    // to get unlocked will block the current thread, which is also responsible
+    // for receiving a call at ConfigStreamServiceImpl::AckManifestIdReceived.
+    // This causes a deadlock and leads to a DeadlineExceeded error.
+    LOG_WARNING("Can not record session content for an empty instance_id.");
+    return;
+  }
+  Session* session = FindSession(instance_id);
+  if (session == nullptr) {
+    LOG_WARNING("Failed to find active session by instrance id: %s",
+                instance_id);
+    return;
+  }
+  session->OnContentSent(byte_count, chunck_count);
+}
+
+void MultiSession::StartHeartBeatCheck() {
+  heartbeat_watcher_ = std::thread([this]() ABSL_LOCKS_EXCLUDED(shutdownMu_) {
+    auto cond = [this]() { return shutdown_; };
+    while (!shutdownMu_.LockWhenWithTimeout(absl::Condition(&cond),
+                                            absl::Minutes(5))) {
+      absl::ReaderMutexLock lock(&sessions_mutex_);
+      for (auto it = sessions_.begin(); it != sessions_.end(); ++it) {
+        it->second->RecordHeartBeatIfChanged();
+      }
+      shutdownMu_.Unlock();
+    }
+    shutdownMu_.Unlock();
+  });
+}
+
+}  // namespace cdc_ft

asset_stream_manager/multi_session.h

@@ -0,0 +1,266 @@
+/*
+ * Copyright 2022 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef ASSET_STREAM_MANAGER_MULTI_SESSION_H_
+#define ASSET_STREAM_MANAGER_MULTI_SESSION_H_
+
+#include <memory>
+#include <string>
+#include <thread>
+#include <unordered_map>
+
+#include "absl/status/status.h"
+#include "absl/status/statusor.h"
+#include "asset_stream_manager/asset_stream_server.h"
+#include "asset_stream_manager/metrics_recorder.h"
+#include "asset_stream_manager/session_config.h"
+#include "common/stopwatch.h"
+#include "data_store/data_store_writer.h"
+#include "manifest/file_chunk_map.h"
+#include "manifest/manifest_updater.h"
+
+namespace cdc_ft {
+
+class ProcessFactory;
+class Session;
+using ManifestUpdatedCb = std::function<void()>;
+
+// Updates the manifest and runs a file watcher in a background thread.
+class MultiSessionRunner {
+ public:
+  // |src_dir| is the source directory on the workstation to stream.
+  // |data_store| can be passed for tests to override the default store used.
+  // |process_factory| abstracts process creation.
+  // |enable_stats| shows whether statistics should be derived.
+  // |wait_duration| is the waiting time for changes in the streamed directory.
+  // |num_updater_threads| is the thread count for the manifest updater.
+  // |manifest_updated_cb| is the callback executed when a new manifest is set.
+  MultiSessionRunner(
+      std::string src_dir, DataStoreWriter* data_store,
+      ProcessFactory* process_factory, bool enable_stats,
+      absl::Duration wait_duration, uint32_t num_updater_threads,
+      MultiSessionMetricsRecorder const* metrics_recorder,
+      ManifestUpdatedCb manifest_updated_cb = ManifestUpdatedCb());
+
+  ~MultiSessionRunner() = default;
+
+  // Starts |server_| of |type| on |port|.
+  absl::Status Initialize(
+      int port, AssetStreamServerType type,
+      ContentSentHandler content_sent = ContentSentHandler());
+
+  // Stops updating the manifest and |server_|.
+  absl::Status Shutdown() ABSL_LOCKS_EXCLUDED(mutex_);
+
+  // Waits until a manifest is ready and the gamelet |instance_id| has
+  // acknowledged the reception of the currently set manifest id. |fuse_timeout|
+  // is the timeout for waiting for the FUSE manifest ack. The time required to
+  // generate the manifest is not part of this timeout as this could take a
+  // longer time for a directory with many files.
+  absl::Status WaitForManifestAck(const std::string& instance_id,
+                                  absl::Duration fuse_timeout);
+
+  absl::Status Status() ABSL_LOCKS_EXCLUDED(mutex_);
+
+  // Returns the current manifest id used by |server_|.
+  ContentIdProto ManifestId() const;
+
+ private:
+  // Updates manifest if the content of the watched directory changes and
+  // distributes it to subscribed gamelets.
+  void Run();
+
+  // Record MultiSessionStart event.
+  void RecordMultiSessionStart(const ManifestUpdater& manifest_updater);
+
+  // Record ManifestUpdate event.
+  void RecordManifestUpdate(const ManifestUpdater& manifest_updater,
+                            absl::Duration duration,
+                            metrics::UpdateTrigger trigger,
+                            absl::Status status);
+
+  void SetStatus(absl::Status status) ABSL_LOCKS_EXCLUDED(mutex_);
+
+  // Files changed callback called from FileWatcherWin.
+  void OnFilesChanged() ABSL_LOCKS_EXCLUDED(mutex_);
+
+  // Directory recreated callback called from FileWatcherWin.
+  void OnDirRecreated() ABSL_LOCKS_EXCLUDED(mutex_);
+
+  // Called during manifest update when the intermediate manifest or the final
+  // manifest is available. Pushes the manifest to connected FUSEs.
+  void SetManifest(const ContentIdProto& manifest_id);
+
+  const std::string src_dir_;
+  DataStoreWriter* const data_store_;
+  ProcessFactory* const process_factory_;
+  FileChunkMap file_chunks_;
+  const absl::Duration wait_duration_;
+  const uint32_t num_updater_threads_;
+  const ManifestUpdatedCb manifest_updated_cb_;
+  std::unique_ptr<AssetStreamServer> server_;
+
+  // Modifications (shutdown, file changes).
+  absl::Mutex mutex_;
+  bool shutdown_ ABSL_GUARDED_BY(mutex_) = false;
+  bool files_changed_ ABSL_GUARDED_BY(mutex_) = false;
+  bool dir_recreated_ ABSL_GUARDED_BY(mutex_) = false;
+  bool manifest_set_ ABSL_GUARDED_BY(mutex_) = false;
+  Stopwatch files_changed_timer_ ABSL_GUARDED_BY(mutex_);
+  absl::Status status_ ABSL_GUARDED_BY(mutex_);
+
+  // Background thread that watches files and updates the manifest.
+  std::unique_ptr<std::thread> thread_;
+
+  MultiSessionMetricsRecorder const* metrics_recorder_;
+};
+
+// Manages an asset streaming session from a fixed directory on the workstation
+// to an arbitrary number of gamelets.
+class MultiSession {
+ public:
+  // Maximum length of cache path. We must be able to write content hashes into
+  // this path:
+  // <cache path>\01234567890123456789<null terminator> = 260 characters.
+  static constexpr size_t kDefaultMaxCachePathLen =
+      260 - 1 - ContentId::kHashSize * 2 - 1;
+
+  // Length of the hash appended to the cache directory, exposed for testing.
+  static constexpr size_t kDirHashLen = 8;
+
+  // |src_dir| is the source directory on the workstation to stream.
+  // |cfg| contains generic configuration parameters for each session.
+  // |process_factory| abstracts process creation.
+  // |data_store| can be passed for tests to override the default store used.
+  // By default, the class uses a DiskDataStore that writes to
+  // %APPDATA%\GGP\asset_streaming|<dir_derived_from_src_dir> on Windows.
+  MultiSession(std::string src_dir, SessionConfig cfg,
+               ProcessFactory* process_factory,
+               MultiSessionMetricsRecorder const* metrics_recorder,
+               std::unique_ptr<DataStoreWriter> data_store = nullptr);
+  ~MultiSession();
+
+  // Initializes the data store if not overridden in the constructor and starts
+  // a background thread for updating the manifest and watching file changes.
+  // Does not wait for the initial manifest update to finish. Use IsRunning()
+  // to determine whether it is finished.
+  // Not thread-safe.
+  absl::Status Initialize();
+
+  // Stops all sessions and shuts down the server.
+  // Not thread-safe.
+  absl::Status Shutdown() ABSL_LOCKS_EXCLUDED(shutdownMu_);
+
+  // Returns the |src_dir| streaming directory passed to the constructor.
+  const std::string& src_dir() const { return src_dir_; }
+
+  // Returns the status of the background thread.
+  // Not thread-safe.
+  absl::Status Status();
+
+  // Starts a new streaming session to the instance with given |instance_id| and
+  // waits until the FUSE has received the initial manifest id.
+  // Returns an error if a session for that instance already exists.
+  // |instance_id| is the instance id of the target remote instance.
+  // |project_id| is id of the project that contains the instance.
+  // |organization_id| is id of the organization that contains the instance.
+  // |instance_ip| is the IP address of the instance.
+  // |instance_port| is the SSH port for connecting to the remote instance.
+  // Thread-safe.
+  absl::Status StartSession(const std::string& instance_id,
+                            const std::string& project_id,
+                            const std::string& organization_id,
+                            const std::string& instance_ip,
+                            uint16_t instance_port)
+      ABSL_LOCKS_EXCLUDED(sessions_mutex_);
+
+  // Starts a new streaming session to the gamelet with given |instance_id|.
+  // Returns a NotFound error if a session for that instance does not exists.
+  // Thread-safe.
+  absl::Status StopSession(const std::string& instance_id)
+      ABSL_LOCKS_EXCLUDED(sessions_mutex_);
+
+  // Returns true if there is an existing session for |instance_id|.
+  bool HasSessionForInstance(const std::string& instance_id)
+      ABSL_LOCKS_EXCLUDED(sessions_mutex_);
+
+  // Returns true if the FUSE process is up and running for an existing session
+  // with ID |instance_id|.
+  bool IsSessionHealthy(const std::string& instance_id)
+      ABSL_LOCKS_EXCLUDED(sessions_mutex_);
+
+  // Returns true if the MultiSession does not have any active sessions.
+  bool Empty() ABSL_LOCKS_EXCLUDED(sessions_mutex_);
+
+  // Returns the number of avtive sessions.
+  uint32_t GetSessionCount() ABSL_LOCKS_EXCLUDED(sessions_mutex_);
+
+  // For a given source directory |dir|, e.g. "C:\path\to\game", returns a
+  // sanitized version of |dir| plus a hash of |dir|, e.g.
+  // "c__path_to_game_abcdef01".
+  static std::string GetCacheDir(std::string dir);
+
+  // Returns the directory where manifest chunks are cached, e.g.
+  // "%APPDATA%\GGP\asset_streaming\c__path_to_game_abcdef01" for
+  // "C:\path\to\game".
+  // The returned path is shortened to |max_len| by removing UTF8 code points
+  // from the beginning of the actual cache directory (c__path...) if necessary.
+  static absl::StatusOr<std::string> GetCachePath(
+      const std::string& src_dir, size_t max_len = kDefaultMaxCachePathLen);
+
+  // Record an event associated with the multi-session.
+  void RecordMultiSessionEvent(metrics::DeveloperLogEvent event,
+                               metrics::EventType code);
+
+  // Record an event for a session associated with the |instance|.
+  void RecordSessionEvent(metrics::DeveloperLogEvent event,
+                          metrics::EventType code,
+                          const std::string& instance_id);
+
+ private:
+  std::string src_dir_;
+  SessionConfig cfg_;
+  ProcessFactory* const process_factory_;
+  std::unique_ptr<DataStoreWriter> data_store_;
+  std::thread heartbeat_watcher_;
+  absl::Mutex shutdownMu_;
+  bool shutdown_ ABSL_GUARDED_BY(shutdownMu_) = false;
+
+  // Background thread for watching file changes and updating the manifest.
+  std::unique_ptr<MultiSessionRunner> runner_;
+
+  // Local forwarding port for the asset stream service.
+  int local_asset_stream_port_ = 0;
+
+  // Maps instance id to sessions.
+  std::unordered_map<std::string, std::unique_ptr<Session>> sessions_
+      ABSL_GUARDED_BY(sessions_mutex_);
+  absl::Mutex sessions_mutex_;
+
+  MultiSessionMetricsRecorder const* metrics_recorder_;
+
+  Session* FindSession(const std::string& instance_id)
+      ABSL_LOCKS_EXCLUDED(sessions_mutex_);
+
+  void OnContentSent(size_t byte_count, size_t chunck_count,
+                     std::string instance_id);
+
+  void StartHeartBeatCheck();
+};
+
+}  // namespace cdc_ft
+
+#endif  // ASSET_STREAM_MANAGER_MULTI_SESSION_H_

asset_stream_manager/multi_session_test.cc

@@ -0,0 +1,488 @@
+// Copyright 2022 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "asset_stream_manager/multi_session.h"
+
+#include <chrono>
+#include <string>
+#include <thread>
+#include <vector>
+
+#include "absl/strings/match.h"
+#include "asset_stream_manager/testing_asset_stream_server.h"
+#include "common/path.h"
+#include "common/platform.h"
+#include "common/process.h"
+#include "common/status_test_macros.h"
+#include "common/test_main.h"
+#include "gtest/gtest.h"
+#include "manifest/manifest_test_base.h"
+
+namespace cdc_ft {
+namespace {
+constexpr char kTestDir[] = "multisession_test_dir";
+constexpr char kData[] = {10, 20, 30, 40, 50, 60, 70, 80, 90};
+constexpr size_t kDataSize = sizeof(kData);
+constexpr char kInstance[] = "test_instance";
+constexpr int kPort = 44444;
+constexpr absl::Duration kTimeout = absl::Milliseconds(5);
+constexpr char kVeryLongPath[] =
+    "C:\\this\\is\\some\\really\\really\\really\\really\\really\\really\\really"
+    "\\really\\really\\really\\really\\really\\really\\really\\really\\really"
+    "\\really\\really\\really\\really\\really\\really\\really\\really\\really"
+    "\\really\\really\\really\\really\\really\\really\\really\\really\\really"
+    "\\really\\long\\path";
+constexpr uint32_t kNumThreads = 1;
+
+struct MetricsRecord {
+  MetricsRecord(metrics::DeveloperLogEvent evt, metrics::EventType code)
+      : evt(std::move(evt)), code(code) {}
+  metrics::DeveloperLogEvent evt;
+  metrics::EventType code;
+};
+
+class MetricsServiceForTest : public MultiSessionMetricsRecorder {
+ public:
+  MetricsServiceForTest() : MultiSessionMetricsRecorder(nullptr) {}
+
+  virtual ~MetricsServiceForTest() = default;
+
+  void RecordEvent(metrics::DeveloperLogEvent event,
+                   metrics::EventType code) const override
+      ABSL_LOCKS_EXCLUDED(mutex_) {
+    absl::MutexLock lock(&mutex_);
+    metrics_records_.push_back(MetricsRecord(std::move(event), code));
+  }
+
+  std::vector<MetricsRecord> GetEventsAndClear(metrics::EventType type)
+      ABSL_LOCKS_EXCLUDED(mutex_) {
+    std::vector<MetricsRecord> events;
+    std::vector<MetricsRecord> remaining;
+    absl::MutexLock lock(&mutex_);
+    for (size_t i = 0; i < metrics_records_.size(); ++i) {
+      if (metrics_records_[i].code == type) {
+        events.push_back(std::move(metrics_records_[i]));
+      } else {
+        remaining.push_back(std::move(metrics_records_[i]));
+      }
+    }
+    metrics_records_ = std::move(remaining);
+    return events;
+  }
+
+ private:
+  mutable absl::Mutex mutex_;
+  mutable std::vector<MetricsRecord> metrics_records_;
+};
+
+class MultiSessionTest : public ManifestTestBase {
+ public:
+  MultiSessionTest() : ManifestTestBase(GetTestDataDir("multi_session")) {
+    Log::Initialize(std::make_unique<ConsoleLog>(LogLevel::kInfo));
+  }
+  ~MultiSessionTest() { Log::Shutdown(); }
+
+  void SetUp() override {
+    // Use a temporary directory to be able to test empty directories (git does
+    // not index empty directories) and creation/deletion of files.
+    EXPECT_OK(path::RemoveDirRec(test_dir_path_));
+    EXPECT_OK(path::CreateDirRec(test_dir_path_));
+    metrics_service_ = new MetricsServiceForTest();
+  }
+
+  void TearDown() override {
+    EXPECT_OK(path::RemoveDirRec(test_dir_path_));
+    delete metrics_service_;
+  }
+
+ protected:
+  // Callback if the manifest was updated == a new manifest is set.
+  void OnManifestUpdated() ABSL_LOCKS_EXCLUDED(mutex_) {
+    absl::MutexLock lock(&mutex_);
+    ++num_manifest_updates_;
+  }
+
+  // Waits until the manifest is fully computed: the manifest id is not changed
+  // anymore.
+  bool WaitForManifestUpdated(uint32_t exp_num_manifest_updates,
+                              absl::Duration timeout = absl::Seconds(5)) {
+    absl::MutexLock lock(&mutex_);
+    auto cond = [&]() {
+      return exp_num_manifest_updates == num_manifest_updates_;
+    };
+    mutex_.AwaitWithTimeout(absl::Condition(&cond), timeout);
+    return exp_num_manifest_updates == num_manifest_updates_;
+  }
+
+  void CheckMultiSessionStartNotRecorded() {
+    std::vector<MetricsRecord> events = metrics_service_->GetEventsAndClear(
+        metrics::EventType::kMultiSessionStart);
+    EXPECT_EQ(events.size(), 0);
+  }
+
+  void CheckMultiSessionStartRecorded(uint64_t byte_count, uint64_t chunk_count,
+                                      uint32_t file_count) {
+    std::vector<MetricsRecord> events = metrics_service_->GetEventsAndClear(
+        metrics::EventType::kMultiSessionStart);
+    ASSERT_EQ(events.size(), 1);
+    metrics::MultiSessionStartData* data =
+        events[0].evt.as_manager_data->multi_session_start_data.get();
+    EXPECT_EQ(data->byte_count, byte_count);
+    EXPECT_EQ(data->chunk_count, chunk_count);
+    EXPECT_EQ(data->file_count, file_count);
+    EXPECT_EQ(data->min_chunk_size, 128 << 10);
+    EXPECT_EQ(data->avg_chunk_size, 256 << 10);
+    EXPECT_EQ(data->max_chunk_size, 1024 << 10);
+  }
+
+  metrics::ManifestUpdateData GetManifestUpdateData(
+      metrics::UpdateTrigger trigger, absl::StatusCode status,
+      size_t total_assets_added_or_updated, size_t total_assets_deleted,
+      size_t total_chunks, size_t total_files_added_or_updated,
+      size_t total_files_failed, size_t total_processed_bytes) {
+    metrics::ManifestUpdateData manifest_upd;
+    manifest_upd.trigger = trigger;
+    manifest_upd.status = status;
+    manifest_upd.total_assets_added_or_updated = total_assets_added_or_updated;
+    manifest_upd.total_assets_deleted = total_assets_deleted;
+    manifest_upd.total_chunks = total_chunks;
+    manifest_upd.total_files_added_or_updated = total_files_added_or_updated;
+    manifest_upd.total_files_failed = total_files_failed;
+    manifest_upd.total_processed_bytes = total_processed_bytes;
+    return manifest_upd;
+  }
+
+  void CheckManifestUpdateRecorded(
+      std::vector<metrics::ManifestUpdateData> manifests) {
+    std::vector<MetricsRecord> events = metrics_service_->GetEventsAndClear(
+        metrics::EventType::kManifestUpdated);
+    ASSERT_EQ(events.size(), manifests.size());
+    for (size_t i = 0; i < manifests.size(); ++i) {
+      metrics::ManifestUpdateData* data =
+          events[i].evt.as_manager_data->manifest_update_data.get();
+      EXPECT_LT(data->local_duration_ms, 60000ull);
+      manifests[i].local_duration_ms = data->local_duration_ms;
+      EXPECT_EQ(*data, manifests[i]);
+    }
+  }
+
+  const std::string test_dir_path_ = path::Join(path::GetTempDir(), kTestDir);
+  WinProcessFactory process_factory_;
+  absl::Mutex mutex_;
+  uint32_t num_manifest_updates_ ABSL_GUARDED_BY(mutex_) = 0;
+  MetricsServiceForTest* metrics_service_;
+};
+
+constexpr char kCacheDir[] = "c__path_to_dir_ee54bbbc";
+
+TEST_F(MultiSessionTest, GetCacheDir_IgnoresTrailingPathSeparators) {
+  EXPECT_EQ(MultiSession::GetCacheDir("C:\\path\\to\\dir"), kCacheDir);
+  EXPECT_EQ(MultiSession::GetCacheDir("C:\\path\\to\\dir\\"), kCacheDir);
+}
+
+TEST_F(MultiSessionTest, GetCacheDir_WorksWithForwardSlashes) {
+  EXPECT_EQ(MultiSession::GetCacheDir("C:/path/to/dir"), kCacheDir);
+  EXPECT_EQ(MultiSession::GetCacheDir("C:/path/to/dir/"), kCacheDir);
+}
+
+TEST_F(MultiSessionTest, GetCacheDir_ReplacesInvalidCharacters) {
+  EXPECT_EQ(MultiSession::GetCacheDir("C:\\<>:\"/\\|?*"),
+            "c___________ae188efd");
+}
+
+TEST_F(MultiSessionTest, GetCacheDir_UsesFullPath) {
+  EXPECT_EQ(MultiSession::GetCacheDir("foo/bar"),
+            MultiSession::GetCacheDir(path::GetFullPath("foo/bar")));
+}
+
+#if PLATFORM_WINDOWS
+TEST_F(MultiSessionTest, GetCacheDir_IgnoresCaseOnWindows) {
+  EXPECT_EQ(MultiSession::GetCacheDir("C:\\PATH\\TO\\DIR"), kCacheDir);
+}
+#endif
+
+TEST_F(MultiSessionTest, GetCachePath_ContainsExpectedParts) {
+  absl::StatusOr<std::string> cache_path =
+      MultiSession::GetCachePath("C:\\path\\to\\dir");
+  ASSERT_OK(cache_path);
+  EXPECT_TRUE(absl::EndsWith(*cache_path, kCacheDir)) << *cache_path;
+  EXPECT_TRUE(
+      absl::StrContains(*cache_path, path::Join("GGP", "asset_streaming")))
+      << *cache_path;
+}
+
+TEST_F(MultiSessionTest, GetCachePath_ShortensLongPaths) {
+  EXPECT_GT(strlen(kVeryLongPath), MultiSession::kDefaultMaxCachePathLen);
+  std::string cache_dir = MultiSession::GetCacheDir(kVeryLongPath);
+  absl::StatusOr<std::string> cache_path =
+      MultiSession::GetCachePath(kVeryLongPath);
+  ASSERT_OK(cache_path);
+  EXPECT_EQ(cache_path->size(), MultiSession::kDefaultMaxCachePathLen);
+  EXPECT_TRUE(
+      absl::StrContains(*cache_path, path::Join("GGP", "asset_streaming")))
+      << *cache_path;
+  // The hash in the end of the path is kept and not shortened.
+  EXPECT_EQ(cache_dir.substr(cache_dir.size() - MultiSession::kDirHashLen),
+            cache_path->substr(cache_path->size() - MultiSession::kDirHashLen));
+}
+
+TEST_F(MultiSessionTest, GetCachePath_DoesNotSplitUtfCodePoints) {
+  // Find out the length of the %APPDATA%\GGP\asset_streaming\" + hash part.
+  absl::StatusOr<std::string> cache_path = MultiSession::GetCachePath("");
+  ASSERT_OK(cache_path);
+  size_t base_len = cache_path->size();
+
+  // Path has are two 2-byte characters. They should not be split in the middle.
+  cache_path = MultiSession::GetCachePath(u8"\u0200\u0200", base_len);
+  ASSERT_OK(cache_path);
+  EXPECT_EQ(cache_path->size(), base_len);
+
+  // %APPDATA%\GGP\asset_streaming\abcdefg
+  cache_path = MultiSession::GetCachePath(u8"\u0200\u0200", base_len + 1);
+  ASSERT_OK(cache_path);
+  EXPECT_EQ(cache_path->size(), base_len);
+
+  // %APPDATA%\GGP\asset_streaming\\u0200abcdefg
+  cache_path = MultiSession::GetCachePath(u8"\u0200\u0200", base_len + 2);
+  ASSERT_OK(cache_path);
+  EXPECT_EQ(cache_path->size(), base_len + 2);
+
+  // %APPDATA%\GGP\asset_streaming\\u0200abcdefg
+  cache_path = MultiSession::GetCachePath(u8"\u0200\u0200", base_len + 3);
+  ASSERT_OK(cache_path);
+  EXPECT_EQ(cache_path->size(), base_len + 2);
+}
+
+// Calculate manifest for an empty directory.
+TEST_F(MultiSessionTest, MultiSessionRunnerOnEmpty) {
+  cfg_.src_dir = test_dir_path_;
+  MultiSessionRunner runner(cfg_.src_dir, &data_store_, &process_factory_,
+                            false /*enable_stats*/, kTimeout, kNumThreads,
+                            metrics_service_,
+                            [this]() { OnManifestUpdated(); });
+  EXPECT_OK(runner.Initialize(kPort, AssetStreamServerType::kTest));
+  EXPECT_OK(runner.WaitForManifestAck(kInstance, kTimeout));
+  // The first update is always the empty manifest, wait for the second one.
+  ASSERT_TRUE(WaitForManifestUpdated(2));
+  ASSERT_NO_FATAL_FAILURE(ExpectManifestEquals({}, runner.ManifestId()));
+  CheckMultiSessionStartRecorded(0, 0, 0);
+  CheckManifestUpdateRecorded(std::vector<metrics::ManifestUpdateData>{
+      GetManifestUpdateData(metrics::UpdateTrigger::kInitUpdateAll,
+                            absl::StatusCode::kOk, 0, 0, 0, 0, 0, 0)});
+
+  EXPECT_OK(runner.Status());
+  EXPECT_OK(runner.Shutdown());
+}
+
+// Calculate manifest for a non-empty directory.
+TEST_F(MultiSessionTest, MultiSessionRunnerNonEmptySucceeds) {
+  // Contains a.txt, subdir/b.txt, subdir/c.txt, subdir/d.txt.
+  cfg_.src_dir = path::Join(base_dir_, "non_empty");
+  MultiSessionRunner runner(cfg_.src_dir, &data_store_, &process_factory_,
+                            false /*enable_stats*/, kTimeout, kNumThreads,
+                            metrics_service_,
+                            [this]() { OnManifestUpdated(); });
+  EXPECT_OK(runner.Initialize(kPort, AssetStreamServerType::kTest));
+  EXPECT_OK(runner.WaitForManifestAck(kInstance, kTimeout));
+  // The first update is always the empty manifest, wait for the second one.
+  ASSERT_TRUE(WaitForManifestUpdated(2));
+  CheckMultiSessionStartRecorded(46, 4, 4);
+  ASSERT_NO_FATAL_FAILURE(ExpectManifestEquals(
+      {"a.txt", "subdir", "subdir/b.txt", "subdir/c.txt", "subdir/d.txt"},
+      runner.ManifestId()));
+  EXPECT_OK(runner.Status());
+  EXPECT_OK(runner.Shutdown());
+}
+
+// Update manifest on adding a file.
+TEST_F(MultiSessionTest, MultiSessionRunnerAddFileSucceeds) {
+  cfg_.src_dir = test_dir_path_;
+  MultiSessionRunner runner(cfg_.src_dir, &data_store_, &process_factory_,
+                            false /*enable_stats*/, kTimeout, kNumThreads,
+                            metrics_service_,
+                            [this]() { OnManifestUpdated(); });
+  EXPECT_OK(runner.Initialize(kPort, AssetStreamServerType::kTest));
+  EXPECT_OK(runner.WaitForManifestAck(kInstance, kTimeout));
+  // The first update is always the empty manifest, wait for the second one.
+  ASSERT_TRUE(WaitForManifestUpdated(2));
+  ASSERT_OK(runner.Status());
+  CheckMultiSessionStartRecorded(0, 0, 0);
+  ASSERT_NO_FATAL_FAILURE(ExpectManifestEquals({}, runner.ManifestId()));
+  CheckManifestUpdateRecorded(std::vector<metrics::ManifestUpdateData>{
+      GetManifestUpdateData(metrics::UpdateTrigger::kInitUpdateAll,
+                            absl::StatusCode::kOk, 0, 0, 0, 0, 0, 0)});
+
+  const std::string file_path = path::Join(test_dir_path_, "file.txt");
+  EXPECT_OK(path::WriteFile(file_path, kData, kDataSize));
+  // 1 file was added = incremented exp_num_manifest_updates.
+  ASSERT_TRUE(WaitForManifestUpdated(3));
+  ASSERT_NO_FATAL_FAILURE(
+      ExpectManifestEquals({"file.txt"}, runner.ManifestId()));
+  CheckMultiSessionStartNotRecorded();
+  CheckManifestUpdateRecorded(std::vector<metrics::ManifestUpdateData>{
+      GetManifestUpdateData(metrics::UpdateTrigger::kRegularUpdate,
+                            absl::StatusCode::kOk, 1, 0, 1, 1, 0, kDataSize)});
+
+  EXPECT_OK(runner.Status());
+  EXPECT_OK(runner.Shutdown());
+}
+
+// Fail if the directory does not exist as the watching could not be started.
+// At this moment we expect that the directory exists.
+TEST_F(MultiSessionTest, MultiSessionRunnerNoDirFails) {
+  cfg_.src_dir = path::Join(base_dir_, "non_existing");
+  MultiSessionRunner runner(cfg_.src_dir, &data_store_, &process_factory_,
+                            false /*enable_stats*/, kTimeout, kNumThreads,
+                            metrics_service_,
+                            [this]() { OnManifestUpdated(); });
+  EXPECT_OK(runner.Initialize(kPort, AssetStreamServerType::kTest));
+
+  ASSERT_FALSE(
+      absl::IsNotFound(runner.WaitForManifestAck(kInstance, kTimeout)));
+  ASSERT_FALSE(WaitForManifestUpdated(1, absl::Milliseconds(10)));
+  CheckMultiSessionStartNotRecorded();
+  CheckManifestUpdateRecorded(std::vector<metrics::ManifestUpdateData>{});
+  EXPECT_NOT_OK(runner.Shutdown());
+  EXPECT_TRUE(absl::StrContains(runner.Status().ToString(),
+                                "Could not start watching"));
+}
+
+// Do not break if the directory is recreated.
+TEST_F(MultiSessionTest, MultiSessionRunnerDirRecreatedSucceeds) {
+  cfg_.src_dir = test_dir_path_;
+  EXPECT_OK(path::WriteFile(path::Join(test_dir_path_, "file.txt"), kData,
+                            kDataSize));
+
+  MultiSessionRunner runner(cfg_.src_dir, &data_store_, &process_factory_,
+                            false /*enable_stats*/, kTimeout, kNumThreads,
+                            metrics_service_,
+                            [this]() { OnManifestUpdated(); });
+  EXPECT_OK(runner.Initialize(kPort, AssetStreamServerType::kTest));
+
+  {
+    SCOPED_TRACE("Originally, only the streamed directory contains file.txt.");
+    EXPECT_OK(runner.WaitForManifestAck(kInstance, kTimeout));
+    // The first update is always the empty manifest, wait for the second one.
+    ASSERT_TRUE(WaitForManifestUpdated(2));
+    CheckMultiSessionStartRecorded((uint64_t)kDataSize, 1, 1);
+    ASSERT_NO_FATAL_FAILURE(
+        ExpectManifestEquals({"file.txt"}, runner.ManifestId()));
+    CheckManifestUpdateRecorded(
+        std::vector<metrics::ManifestUpdateData>{GetManifestUpdateData(
+            metrics::UpdateTrigger::kInitUpdateAll, absl::StatusCode::kOk, 1, 0,
+            1, 1, 0, kDataSize)});
+  }
+
+  {
+    SCOPED_TRACE(
+        "Remove the streamed directory, the manifest should become empty.");
+    EXPECT_OK(path::RemoveDirRec(test_dir_path_));
+    ASSERT_TRUE(WaitForManifestUpdated(3));
+    ASSERT_NO_FATAL_FAILURE(ExpectManifestEquals({}, runner.ManifestId()));
+    CheckManifestUpdateRecorded(
+        std::vector<metrics::ManifestUpdateData>{GetManifestUpdateData(
+            metrics::UpdateTrigger::kRunningUpdateAll,
+            absl::StatusCode::kNotFound, 1, 0, 1, 1, 0, kDataSize)});
+  }
+
+  {
+    SCOPED_TRACE(
+        "Create the watched directory -> an empty manifest should be "
+        "streamed.");
+    EXPECT_OK(path::CreateDirRec(test_dir_path_));
+    EXPECT_TRUE(WaitForManifestUpdated(4));
+    ASSERT_NO_FATAL_FAILURE(ExpectManifestEquals({}, runner.ManifestId()));
+    CheckManifestUpdateRecorded(std::vector<metrics::ManifestUpdateData>{
+        GetManifestUpdateData(metrics::UpdateTrigger::kRunningUpdateAll,
+                              absl::StatusCode::kOk, 0, 0, 0, 0, 0, 0)});
+  }
+
+  {
+    SCOPED_TRACE("Create 'new_file.txt' -> new manifest should be created.");
+    EXPECT_OK(path::WriteFile(path::Join(test_dir_path_, "new_file.txt"), kData,
+                              kDataSize));
+    ASSERT_TRUE(WaitForManifestUpdated(5));
+    ASSERT_NO_FATAL_FAILURE(
+        ExpectManifestEquals({"new_file.txt"}, runner.ManifestId()));
+    CheckManifestUpdateRecorded(
+        std::vector<metrics::ManifestUpdateData>{GetManifestUpdateData(
+            metrics::UpdateTrigger::kRegularUpdate, absl::StatusCode::kOk, 1, 0,
+            1, 1, 0, kDataSize)});
+    CheckMultiSessionStartNotRecorded();
+  }
+
+  EXPECT_OK(runner.Status());
+  EXPECT_OK(runner.Shutdown());
+}
+
+// Fail if the streamed source is a file.
+TEST_F(MultiSessionTest, MultiSessionRunnerFileAsStreamedDirFails) {
+  cfg_.src_dir = path::Join(test_dir_path_, "file.txt");
+  EXPECT_OK(path::WriteFile(cfg_.src_dir, kData, kDataSize));
+
+  MultiSessionRunner runner(cfg_.src_dir, &data_store_, &process_factory_,
+                            false /*enable_stats*/, kTimeout, kNumThreads,
+                            metrics_service_,
+                            [this]() { OnManifestUpdated(); });
+  EXPECT_OK(runner.Initialize(kPort, AssetStreamServerType::kTest));
+  ASSERT_FALSE(WaitForManifestUpdated(1, absl::Milliseconds(10)));
+  CheckMultiSessionStartNotRecorded();
+  CheckManifestUpdateRecorded(std::vector<metrics::ManifestUpdateData>{});
+  EXPECT_NOT_OK(runner.Shutdown());
+  EXPECT_TRUE(absl::StrContains(runner.Status().ToString(),
+                                "Failed to update manifest"))
+      << runner.Status().ToString();
+}
+
+// Stream an empty manifest if the streamed directory was re-created as a file.
+TEST_F(MultiSessionTest,
+       MultiSessionRunnerDirRecreatedAsFileSucceedsWithEmptyManifest) {
+  cfg_.src_dir = path::Join(test_dir_path_, "file");
+  EXPECT_OK(path::CreateDirRec(cfg_.src_dir));
+
+  MultiSessionRunner runner(cfg_.src_dir, &data_store_, &process_factory_,
+                            false /*enable_stats*/, kTimeout, kNumThreads,
+                            metrics_service_,
+                            [this]() { OnManifestUpdated(); });
+  EXPECT_OK(runner.Initialize(kPort, AssetStreamServerType::kTest));
+  ASSERT_TRUE(WaitForManifestUpdated(2));
+  CheckMultiSessionStartRecorded(0, 0, 0);
+  CheckManifestUpdateRecorded(std::vector<metrics::ManifestUpdateData>{
+      GetManifestUpdateData(metrics::UpdateTrigger::kInitUpdateAll,
+                            absl::StatusCode::kOk, 0, 0, 0, 0, 0, 0)});
+  ASSERT_NO_FATAL_FAILURE(ExpectManifestEquals({}, runner.ManifestId()));
+
+  // Remove the streamed directory, the manifest should become empty.
+  EXPECT_OK(path::RemoveDirRec(cfg_.src_dir));
+  ASSERT_TRUE(WaitForManifestUpdated(3));
+  ASSERT_NO_FATAL_FAILURE(ExpectManifestEquals({}, runner.ManifestId()));
+  CheckManifestUpdateRecorded(std::vector<metrics::ManifestUpdateData>{
+      GetManifestUpdateData(metrics::UpdateTrigger::kRunningUpdateAll,
+                            absl::StatusCode::kNotFound, 0, 0, 0, 0, 0, 0)});
+
+  EXPECT_OK(path::WriteFile(cfg_.src_dir, kData, kDataSize));
+  EXPECT_TRUE(WaitForManifestUpdated(4));
+  ASSERT_NO_FATAL_FAILURE(ExpectManifestEquals({}, runner.ManifestId()));
+  CheckManifestUpdateRecorded(
+      std::vector<metrics::ManifestUpdateData>{GetManifestUpdateData(
+          metrics::UpdateTrigger::kRunningUpdateAll,
+          absl::StatusCode::kFailedPrecondition, 0, 0, 0, 0, 0, 0)});
+  CheckMultiSessionStartNotRecorded();
+
+  EXPECT_OK(runner.Status());
+  EXPECT_OK(runner.Shutdown());
+}
+
+}  // namespace
+}  // namespace cdc_ft

asset_stream_manager/session.cc

@@ -0,0 +1,131 @@
+// Copyright 2022 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "asset_stream_manager/session.h"
+
+#include "asset_stream_manager/cdc_fuse_manager.h"
+#include "common/log.h"
+#include "common/port_manager.h"
+#include "common/status.h"
+#include "common/status_macros.h"
+#include "metrics/enums.h"
+#include "metrics/messages.h"
+
+namespace cdc_ft {
+namespace {
+
+// Timeout for initial gamelet connection.
+constexpr double kInstanceConnectionTimeoutSec = 10.0f;
+
+metrics::DeveloperLogEvent GetEventWithHeartBeatData(size_t bytes,
+                                                     size_t chunks) {
+  metrics::DeveloperLogEvent evt;
+  evt.as_manager_data = std::make_unique<metrics::AssetStreamingManagerData>();
+  evt.as_manager_data->session_data = std::make_unique<metrics::SessionData>();
+  evt.as_manager_data->session_data->byte_count = bytes;
+  evt.as_manager_data->session_data->chunk_count = chunks;
+  return std::move(evt);
+}
+
+}  // namespace
+
+Session::Session(std::string instance_id, std::string instance_ip,
+                 uint16_t instance_port, SessionConfig cfg,
+                 ProcessFactory* process_factory,
+                 std::unique_ptr<SessionMetricsRecorder> metrics_recorder)
+    : instance_id_(std::move(instance_id)),
+      cfg_(std::move(cfg)),
+      process_factory_(process_factory),
+      remote_util_(cfg_.verbosity, cfg_.quiet, process_factory,
+                   /*forward_output_to_logging=*/true),
+      metrics_recorder_(std::move(metrics_recorder)) {
+  assert(metrics_recorder_);
+  remote_util_.SetIpAndPort(instance_ip, instance_port);
+}
+
+Session::~Session() {
+  absl::Status status = Stop();
+  if (!status.ok()) {
+    LOG_ERROR("Failed to stop session for instance '%s': %s", instance_id_,
+              status.ToString());
+  }
+}
+
+absl::Status Session::Start(int local_port, int first_remote_port,
+                            int last_remote_port) {
+  // Find an available remote port.
+  std::unordered_set<int> ports;
+  ASSIGN_OR_RETURN(
+      ports,
+      PortManager::FindAvailableRemotePorts(
+          first_remote_port, last_remote_port, "127.0.0.1", process_factory_,
+          &remote_util_, kInstanceConnectionTimeoutSec, true),
+      "Failed to find an available remote port in the range [%d, %d]",
+      first_remote_port, last_remote_port);
+  assert(!ports.empty());
+  int remote_port = *ports.begin();
+
+  assert(!fuse_);
+  fuse_ = std::make_unique<CdcFuseManager>(instance_id_, process_factory_,
+                                           &remote_util_);
+  RETURN_IF_ERROR(
+      fuse_->Start(local_port, remote_port, cfg_.verbosity, cfg_.fuse_debug,
+                   cfg_.fuse_singlethreaded, cfg_.stats, cfg_.fuse_check,
+                   cfg_.fuse_cache_capacity, cfg_.fuse_cleanup_timeout_sec,
+                   cfg_.fuse_access_idle_timeout_sec),
+      "Failed to start instance component");
+  return absl::OkStatus();
+}
+
+absl::Status Session::Stop() {
+  absl::ReaderMutexLock lock(&transferred_data_mu_);
+  // TODO: Record error on session end.
+  metrics_recorder_->RecordEvent(
+      GetEventWithHeartBeatData(transferred_bytes_, transferred_chunks_),
+      metrics::EventType::kSessionEnd);
+  if (fuse_) {
+    RETURN_IF_ERROR(fuse_->Stop());
+    fuse_.reset();
+  }
+
+  return absl::OkStatus();
+}
+
+bool Session::IsHealthy() { return fuse_->IsHealthy(); }
+
+void Session::RecordEvent(metrics::DeveloperLogEvent event,
+                          metrics::EventType code) const {
+  metrics_recorder_->RecordEvent(std::move(event), code);
+}
+
+void Session::OnContentSent(size_t bytes, size_t chunks) {
+  absl::WriterMutexLock lock(&transferred_data_mu_);
+  transferred_bytes_ += bytes;
+  transferred_chunks_ += chunks;
+}
+
+void Session::RecordHeartBeatIfChanged() {
+  absl::ReaderMutexLock lock(&transferred_data_mu_);
+  if (transferred_bytes_ == last_read_bytes_ &&
+      transferred_chunks_ == last_read_chunks_) {
+    return;
+  }
+  last_read_bytes_ = transferred_bytes_;
+  last_read_chunks_ = transferred_chunks_;
+  metrics_recorder_->RecordEvent(
+      GetEventWithHeartBeatData(last_read_bytes_, last_read_chunks_),
+      metrics::EventType::kSessionHeartBeat);
+}
+
+}  // namespace cdc_ft

asset_stream_manager/session.h

@@ -0,0 +1,90 @@
+/*
+ * Copyright 2022 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef ASSET_STREAM_MANAGER_SESSION_H_
+#define ASSET_STREAM_MANAGER_SESSION_H_
+
+#include <memory>
+#include <string>
+#include <unordered_map>
+
+#include "absl/status/status.h"
+#include "asset_stream_manager/metrics_recorder.h"
+#include "asset_stream_manager/session_config.h"
+#include "common/remote_util.h"
+
+namespace cdc_ft {
+
+class CdcFuseManager;
+class ProcessFactory;
+class Process;
+
+// Manages the connection of a workstation to a single gamelet.
+class Session {
+ public:
+  // |instance_id| is a unique id for the remote instance.
+  // |instance_ip| is the IP address of the remote instance.
+  // |instance_port| is the SSH tunnel port for connecting to the instance.
+  // |cfg| contains generic configuration parameters for the session.
+  // |process_factory| abstracts process creation.
+  Session(std::string instance_id, std::string instance_ip,
+          uint16_t instance_port, SessionConfig cfg,
+          ProcessFactory* process_factory,
+          std::unique_ptr<SessionMetricsRecorder> metrics_recorder);
+  ~Session();
+
+  // Starts the CDC FUSE on the instance with established port forwarding.
+  // |local_port| is the local reverse forwarding port to use.
+  // [|first_remote_port|, |last_remote_port|] are the allowed remote ports.
+  absl::Status Start(int local_port, int first_remote_port,
+                     int last_remote_port);
+
+  // Shuts down the connection to the instance.
+  absl::Status Stop() ABSL_LOCKS_EXCLUDED(transferred_data_mu_);
+
+  // Returns true if the FUSE process is running.
+  bool IsHealthy();
+
+  // Record an event for the session.
+  void RecordEvent(metrics::DeveloperLogEvent event,
+                   metrics::EventType code) const;
+
+  // Is called when content was sent during the session.
+  void OnContentSent(size_t bytes, size_t chunks)
+      ABSL_LOCKS_EXCLUDED(transferred_data_mu_);
+
+  // Records heart beat data if it has changed since last record.
+  void RecordHeartBeatIfChanged() ABSL_LOCKS_EXCLUDED(transferred_data_mu_);
+
+ private:
+  const std::string instance_id_;
+  const SessionConfig cfg_;
+  ProcessFactory* const process_factory_;
+
+  RemoteUtil remote_util_;
+  std::unique_ptr<CdcFuseManager> fuse_;
+  std::unique_ptr<SessionMetricsRecorder> metrics_recorder_;
+
+  absl::Mutex transferred_data_mu_;
+  uint64_t transferred_bytes_ ABSL_GUARDED_BY(transferred_data_mu_) = 0;
+  uint64_t transferred_chunks_ ABSL_GUARDED_BY(transferred_data_mu_) = 0;
+  uint64_t last_read_bytes_ = 0;
+  uint64_t last_read_chunks_ = 0;
+};
+
+}  // namespace cdc_ft
+
+#endif  // ASSET_STREAM_MANAGER_SESSION_H_

asset_stream_manager/session_config.h

@@ -0,0 +1,63 @@
+/*
+ * Copyright 2022 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef ASSET_STREAM_MANAGER_SESSION_CONFIG_H_
+#define ASSET_STREAM_MANAGER_SESSION_CONFIG_H_
+
+#include <cstdint>
+
+namespace cdc_ft {
+
+// The values set in this config do not necessarily denote the default values.
+// For the defaults, see the corresponding flag values.
+struct SessionConfig {
+  // General log verbosity.
+  int verbosity = 0;
+
+  // Silence logs from process execution.
+  bool quiet = false;
+
+  // Print detailed streaming stats.
+  bool stats = false;
+
+  // Whether to run FUSE in debug mode.
+  bool fuse_debug = false;
+
+  // Whether to run FUSE in single-threaded mode.
+  bool fuse_singlethreaded = false;
+
+  // Whether to run FUSE consistency check.
+  bool fuse_check = false;
+
+  // Cache capacity with a suffix.
+  uint64_t fuse_cache_capacity = 0;
+
+  // Cleanup timeout in seconds.
+  uint32_t fuse_cleanup_timeout_sec = 0;
+
+  // Access idling timeout in seconds.
+  uint32_t fuse_access_idle_timeout_sec = 0;
+
+  // Number of threads used in the manifest updater to compute chunks/hashes.
+  uint32_t manifest_updater_threads = 0;
+
+  // Time to wait until running a manifest update after detecting a file change.
+  uint32_t file_change_wait_duration_ms = 0;
+};
+
+}  // namespace cdc_ft
+
+#endif  // ASSET_STREAM_MANAGER_SESSION_CONFIG_H_

asset_stream_manager/session_management_server.cc

@@ -0,0 +1,76 @@
+// Copyright 2022 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "asset_stream_manager/session_management_server.h"
+
+#include "absl/strings/str_format.h"
+#include "asset_stream_manager/background_service_impl.h"
+#include "asset_stream_manager/local_assets_stream_manager_service_impl.h"
+#include "asset_stream_manager/session_manager.h"
+#include "common/log.h"
+#include "common/status.h"
+#include "common/status_macros.h"
+#include "grpcpp/grpcpp.h"
+
+namespace cdc_ft {
+
+SessionManagementServer::SessionManagementServer(
+    grpc::Service* session_service, grpc::Service* background_service,
+    SessionManager* session_manager)
+    : session_service_(session_service),
+      background_service_(background_service),
+      session_manager_(session_manager) {}
+
+SessionManagementServer::~SessionManagementServer() = default;
+
+absl::Status SessionManagementServer::Start(int port) {
+  assert(!server_);
+
+  // Use 127.0.0.1 here to enforce IPv4. Otherwise, if only IPv4 is blocked on
+  // |port|, the server is started with IPv6 only, but clients are connecting
+  // with IPv4.
+  int selected_port = 0;
+  std::string server_address = absl::StrFormat("127.0.0.1:%i", port);
+  grpc::ServerBuilder builder;
+  builder.AddListeningPort(server_address, grpc::InsecureServerCredentials(),
+                           &selected_port);
+  builder.RegisterService(session_service_);
+  builder.RegisterService(background_service_);
+  server_ = builder.BuildAndStart();
+  if (selected_port != port) {
+    return MakeStatus(
+        "Failed to start session management server: Could not listen on port "
+        "%i. Is the port in use?",
+        port);
+  }
+  if (!server_) {
+    return MakeStatus(
+        "Failed to start session management server. Check asset_stream_manager "
+        "logs.");
+  }
+  LOG_INFO("Session management server listening on '%s'", server_address);
+  return absl::OkStatus();
+}
+
+void SessionManagementServer::RunUntilShutdown() { server_->Wait(); }
+
+absl::Status SessionManagementServer::Shutdown() {
+  assert(server_);
+  RETURN_IF_ERROR(session_manager_->Shutdown(),
+                  "Failed to shut down session manager");
+  server_->Shutdown();
+  server_->Wait();
+}
+
+}  // namespace cdc_ft

asset_stream_manager/session_management_server.h

@@ -0,0 +1,63 @@
+/*
+ * Copyright 2022 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef ASSET_STREAM_MANAGER_SESSION_MANAGEMENT_SERVER_H_
+#define ASSET_STREAM_MANAGER_SESSION_MANAGEMENT_SERVER_H_
+
+#include <memory>
+
+#include "absl/status/status.h"
+
+namespace grpc {
+class Server;
+class Service;
+}  // namespace grpc
+
+namespace cdc_ft {
+
+class SessionManager;
+class ProcessFactory;
+
+// gRPC server for managing streaming sessions. Contains these services:
+// - LocalAssetsStreamManager
+// - Background
+class SessionManagementServer {
+ public:
+  SessionManagementServer(grpc::Service* session_service,
+                          grpc::Service* background_service,
+                          SessionManager* session_manager);
+  ~SessionManagementServer();
+
+  // Starts the server on the local port |port|.
+  absl::Status Start(int port);
+
+  // Waits until ProcessManager issues an Exit() request to the background
+  // service.
+  void RunUntilShutdown();
+
+  // Shuts down the session manager and the server.
+  absl::Status Shutdown();
+
+ private:
+  grpc::Service* session_service_;
+  grpc::Service* background_service_;
+  SessionManager* const session_manager_;
+  std::unique_ptr<grpc::Server> server_;
+};
+
+}  // namespace cdc_ft
+
+#endif  // ASSET_STREAM_MANAGER_SESSION_MANAGEMENT_SERVER_H_

asset_stream_manager/session_manager.cc

@@ -0,0 +1,193 @@
+// Copyright 2022 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "asset_stream_manager/session_manager.h"
+
+#include "absl/strings/str_split.h"
+#include "asset_stream_manager/multi_session.h"
+#include "common/log.h"
+#include "common/process.h"
+#include "common/status.h"
+#include "common/status_macros.h"
+#include "manifest/manifest_updater.h"
+
+namespace cdc_ft {
+namespace {
+
+// Returns a key to uniquely map a streaming directory |src_dir| to a
+// MultiSession instance.
+std::string GetMultiSessionKey(const std::string src_dir) {
+  // Use the cache dir as a key to identify MultiSessions. That way, different
+  // representations of the same dir (e.g. dir and dir\) map to the same
+  // MultiSession.
+  return MultiSession::GetCacheDir(src_dir);
+}
+
+}  // namespace
+
+SessionManager::SessionManager(SessionConfig cfg,
+                               ProcessFactory* process_factory,
+                               metrics::MetricsService* metrics_service)
+    : cfg_(cfg),
+      process_factory_(process_factory),
+      metrics_service_(metrics_service) {}
+
+SessionManager::~SessionManager() = default;
+
+absl::Status SessionManager::Shutdown() {
+  absl::MutexLock lock(&sessions_mutex_);
+
+  for (const auto& [key, ms] : sessions_) {
+    LOG_INFO("Shutting down MultiSession for path '%s'", ms->src_dir());
+    RETURN_IF_ERROR(ms->Shutdown(),
+                    "Failed to shut down MultiSession for path '%s'",
+                    ms->src_dir());
+  }
+  sessions_.clear();
+  return absl::OkStatus();
+}
+
+absl::Status SessionManager::StartSession(
+    const std::string& instance_id, const std::string& project_id,
+    const std::string& organization_id, const std::string& instance_ip,
+    uint16_t instance_port, const std::string& src_dir,
+    MultiSession** multi_session, metrics::SessionStartStatus* metrics_status) {
+  *multi_session = nullptr;
+  *metrics_status = metrics::SessionStartStatus::kOk;
+
+  absl::MutexLock lock(&sessions_mutex_);
+
+  // Check if the directory is correct as early as possible.
+  absl::Status status = ManifestUpdater::IsValidDir(src_dir);
+  if (!status.ok()) {
+    absl::Status stop_status = StopSessionInternal(instance_id);
+    if (!stop_status.ok() && !absl::IsNotFound(stop_status)) {
+      LOG_ERROR("Failed to stop previous session for instance '%s': '%s'",
+                instance_id, stop_status.ToString());
+    }
+    *metrics_status = metrics::SessionStartStatus::kInvalidDirError;
+    return WrapStatus(status, "Failed to start session for path '%s'", src_dir);
+  }
+
+  // Early out if we are streaming the workstation dir to the given gamelet.
+  MultiSession* ms = GetMultiSession(src_dir);
+  *multi_session = ms;
+  if (ms && ms->HasSessionForInstance(instance_id)) {
+    if (ms->IsSessionHealthy(instance_id)) {
+      LOG_INFO("Reusing existing session");
+      return absl::OkStatus();
+    }
+
+    LOG_INFO("Existing session for instance '%s' is not healthy. Restarting.",
+             instance_id);
+
+    // We could also fall through, but this might restart the MultiSession.
+    status = ms->StopSession(instance_id);
+    if (status.ok()) {
+      status = ms->StartSession(instance_id, project_id, organization_id,
+                                instance_ip, instance_port);
+    }
+    if (!status.ok()) {
+      *metrics_status = metrics::SessionStartStatus::kRestartSessionError;
+    }
+    return WrapStatus(status, "Failed to restart session for instance '%s'",
+                      instance_id);
+  }
+
+  // If we are already streaming to the given gamelet, but from another
+  // workstation directory, stop that session.
+  // Note that NotFoundError is OK and expected (it means no session exists).
+  status = StopSessionInternal(instance_id);
+  if (!status.ok() && !absl::IsNotFound(status)) {
+    *metrics_status = metrics::SessionStartStatus::kStopSessionError;
+    return WrapStatus(status,
+                      "Failed to stop previous session for instance '%s'",
+                      instance_id);
+  }
+
+  // Get or create the MultiSession for the given workstation directory.
+  absl::StatusOr<MultiSession*> ms_res = GetOrCreateMultiSession(src_dir);
+  if (!ms_res.ok()) {
+    *metrics_status = metrics::SessionStartStatus::kCreateMultiSessionError;
+    return WrapStatus(ms_res.status(),
+                      "Failed to create MultiSession for path '%s'", src_dir);
+  }
+  ms = ms_res.value();
+  *multi_session = ms;
+
+  // Start the session.
+  LOG_INFO("Starting streaming session from path '%s' to instance '%s'",
+           src_dir, instance_id);
+  status = ms->StartSession(instance_id, project_id, organization_id,
+                            instance_ip, instance_port);
+  if (!status.ok()) {
+    *metrics_status = metrics::SessionStartStatus::kStartSessionError;
+  }
+  return status;
+}
+
+absl::Status SessionManager::StopSession(const std::string& instance_id) {
+  absl::MutexLock lock(&sessions_mutex_);
+  return StopSessionInternal(instance_id);
+}
+
+MultiSession* SessionManager::GetMultiSession(const std::string& src_dir) {
+  const std::string key = GetMultiSessionKey(src_dir);
+  SessionMap::iterator iter = sessions_.find(key);
+  return iter != sessions_.end() ? iter->second.get() : nullptr;
+}
+
+absl::StatusOr<MultiSession*> SessionManager::GetOrCreateMultiSession(
+    const std::string& src_dir) {
+  const std::string key = GetMultiSessionKey(src_dir);
+  SessionMap::iterator iter = sessions_.find(key);
+  if (iter == sessions_.end()) {
+    LOG_INFO("Creating new MultiSession for path '%s'", src_dir);
+    auto ms = std::make_unique<MultiSession>(
+        src_dir, cfg_, process_factory_,
+        new MultiSessionMetricsRecorder(metrics_service_));
+    RETURN_IF_ERROR(ms->Initialize(), "Failed to initialize MultiSession");
+    iter = sessions_.insert({key, std::move(ms)}).first;
+  }
+
+  return iter->second.get();
+}
+
+absl::Status SessionManager::StopSessionInternal(const std::string& instance) {
+  absl::Status status;
+  for (const auto& [key, ms] : sessions_) {
+    if (!ms->HasSessionForInstance(instance)) continue;
+
+    LOG_INFO("Stopping session streaming from '%s' to instance '%s'",
+             ms->src_dir(), instance);
+    RETURN_IF_ERROR(ms->StopSession(instance),
+                    "Failed to stop session for instance '%s'", instance);
+
+    // Session was stopped. If the MultiSession is empty now, delete it.
+    if (ms->Empty()) {
+      LOG_INFO("Shutting down MultiSession for path '%s'", ms->src_dir());
+      RETURN_IF_ERROR(ms->Shutdown(),
+                      "Failed to shut down MultiSession for path '%s'",
+                      ms->src_dir());
+      sessions_.erase(key);
+    }
+
+    return absl::OkStatus();
+  }
+
+  return absl::NotFoundError(
+      absl::StrFormat("No session for instance id '%s' found", instance));
+}
+
+}  // namespace cdc_ft

asset_stream_manager/session_manager.h

@@ -0,0 +1,100 @@
+/*
+ * Copyright 2022 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef ASSET_STREAM_MANAGER_SESSION_MANAGER_H_
+#define ASSET_STREAM_MANAGER_SESSION_MANAGER_H_
+
+#include <memory>
+#include <unordered_map>
+
+#include "absl/status/status.h"
+#include "absl/status/statusor.h"
+#include "absl/synchronization/mutex.h"
+#include "asset_stream_manager/session_config.h"
+#include "metrics/metrics.h"
+
+namespace cdc_ft {
+
+class MultiSession;
+class ProcessFactory;
+
+// Implements a service to start and stop streaming sessions as a server.
+// The corresponding clients are implemented by the ggp CLI and SDK Proxy.
+// The CLI triggers StartSession() from `ggp instance mount --local-dir` and
+// StopSession() from `ggp instance unmount`. SDK Proxy invokes StartSession()
+// when a user starts a new game from the partner portal and sets an `Asset
+// streaming directory` in the `Advanced settings` in the `Play settings`
+// dialog.
+// This service is owned by SessionManagementServer.
+class SessionManager {
+ public:
+  SessionManager(SessionConfig cfg, ProcessFactory* process_factory,
+                 metrics::MetricsService* metrics_service);
+  ~SessionManager();
+
+  // Starts a session and populates |multi_session| and |metrics_status|.
+  absl::Status StartSession(const std::string& instance_id,
+                            const std::string& project_id,
+                            const std::string& organization_id,
+                            const std::string& instance_ip,
+                            uint16_t instance_port, const std::string& src_dir,
+                            MultiSession** multi_session,
+                            metrics::SessionStartStatus* metrics_status)
+      ABSL_LOCKS_EXCLUDED(sessions_mutex_);
+
+  // Stops the session for the given |instance|. Returns a NotFound error if no
+  // session exists.
+  absl::Status StopSession(const std::string& instance)
+      ABSL_LOCKS_EXCLUDED(sessions_mutex_);
+
+  // Shuts down all existing MultiSessions.
+  absl::Status Shutdown() ABSL_LOCKS_EXCLUDED(sessions_mutex_);
+
+ private:
+  // Stops the session for the given |instance|. Returns a NotFound error if no
+  // session exists.
+  absl::Status StopSessionInternal(const std::string& instance)
+      ABSL_EXCLUSIVE_LOCKS_REQUIRED(sessions_mutex_);
+
+  // Returns the MultiSession for the given workstation directory |src_dir| or
+  // nullptr if it does not exist.
+  MultiSession* GetMultiSession(const std::string& src_dir)
+      ABSL_EXCLUSIVE_LOCKS_REQUIRED(sessions_mutex_);
+
+  // Gets an existing MultiSession or creates a new one for the given
+  // workstation directory |src_dir|.
+  absl::StatusOr<MultiSession*> GetOrCreateMultiSession(
+      const std::string& src_dir)
+      ABSL_EXCLUSIVE_LOCKS_REQUIRED(sessions_mutex_);
+
+  // Sets session start status for a metrics event.
+  void SetSessionStartStatus(metrics::DeveloperLogEvent* evt,
+                             absl::Status absl_status,
+                             metrics::SessionStartStatus status) const;
+
+  const SessionConfig cfg_;
+  ProcessFactory* const process_factory_;
+  metrics::MetricsService* const metrics_service_;
+
+  absl::Mutex sessions_mutex_;
+  using SessionMap =
+      std::unordered_map<std::string, std::unique_ptr<MultiSession>>;
+  SessionMap sessions_ ABSL_GUARDED_BY(sessions_mutex_);
+};
+
+}  // namespace cdc_ft
+
+#endif  // ASSET_STREAM_MANAGER_SESSION_MANAGER_H_

asset_stream_manager/testdata/multi_session/non_empty/a.txt

@@ -0,0 +1 @@
+aaaaaaaa

asset_stream_manager/testdata/multi_session/non_empty/subdir/b.txt

@@ -0,0 +1 @@
+bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb

asset_stream_manager/testdata/multi_session/non_empty/subdir/c.txt

@@ -0,0 +1 @@
+c

asset_stream_manager/testdata/multi_session/non_empty/subdir/d.txt

@@ -0,0 +1 @@
+d

asset_stream_manager/testing_asset_stream_server.cc

@@ -0,0 +1,50 @@
+// Copyright 2022 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "asset_stream_manager/testing_asset_stream_server.h"
+
+#include "data_store/data_store_reader.h"
+#include "manifest/file_chunk_map.h"
+
+namespace cdc_ft {
+
+TestingAssetStreamServer::TestingAssetStreamServer(
+    std::string src_dir, DataStoreReader* data_store_reader,
+    FileChunkMap* file_chunks)
+    : AssetStreamServer(src_dir, data_store_reader, file_chunks) {}
+
+TestingAssetStreamServer::~TestingAssetStreamServer() = default;
+
+absl::Status TestingAssetStreamServer::Start(int port) {
+  return absl::OkStatus();
+}
+
+void TestingAssetStreamServer::SetManifestId(
+    const ContentIdProto& manifest_id) {
+  absl::MutexLock lock(&mutex_);
+  manifest_id_ = manifest_id;
+}
+
+absl::Status TestingAssetStreamServer::WaitForManifestAck(
+    const std::string& instance, absl::Duration timeout) {
+  return absl::OkStatus();
+}
+
+void TestingAssetStreamServer::Shutdown() {}
+
+ContentIdProto TestingAssetStreamServer::GetManifestId() const {
+  absl::MutexLock lock(&mutex_);
+  return manifest_id_;
+}
+}  // namespace cdc_ft

asset_stream_manager/testing_asset_stream_server.h

@@ -0,0 +1,60 @@
+/*
+ * Copyright 2022 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef ASSET_STREAM_MANAGER_TESTING_ASSET_STREAM_SERVER_H_
+#define ASSET_STREAM_MANAGER_TESTING_ASSET_STREAM_SERVER_H_
+
+#include <memory>
+#include <string>
+
+#include "absl/base/thread_annotations.h"
+#include "absl/status/status.h"
+#include "absl/synchronization/mutex.h"
+#include "asset_stream_manager/grpc_asset_stream_server.h"
+#include "manifest/manifest_proto_defs.h"
+
+namespace cdc_ft {
+
+// Not thread-safe testing server for streaming assets.
+class TestingAssetStreamServer : public AssetStreamServer {
+ public:
+  TestingAssetStreamServer(std::string src_dir,
+                           DataStoreReader* data_store_reader,
+                           FileChunkMap* file_chunks);
+
+  ~TestingAssetStreamServer();
+
+  // AssetStreamServer:
+
+  absl::Status Start(int port) override;
+
+  void SetManifestId(const ContentIdProto& manifest_id)
+      ABSL_LOCKS_EXCLUDED(mutex_) override;
+
+  absl::Status WaitForManifestAck(const std::string& instance,
+                                  absl::Duration timeout) override;
+  void Shutdown() override;
+
+  ContentIdProto GetManifestId() const ABSL_LOCKS_EXCLUDED(mutex_) override;
+
+ private:
+  mutable absl::Mutex mutex_;
+  ContentIdProto manifest_id_ ABSL_GUARDED_BY(mutex_);
+};
+
+}  // namespace cdc_ft
+
+#endif  // ASSET_STREAM_MANAGER_TESTING_ASSET_STREAM_SERVER_H_