mirror of
https://github.com/nestriness/cdc-file-transfer.git
synced 2026-01-30 14:45:37 +02:00
76bbdb01bb
This change introduces dynamic manifest updates to asset streaming. Asset streaming describes the directory to be streamed in a manifest, which is a proto definition of all content metadata. This information is sufficient to answer `stat` and `readdir` calls in the FUSE layer without additional round-trips to the workstation. When a directory is streamed for the first time, the corresponding manifest is created in two steps: 1. The directory is traversed recursively and the inode information of all contained files and directories is written to the manifest. 2. The content of all identified files is processed to generate each file's chunk list. This list is part of the definition of a file in the manifest. * The chunk boundaries are identified using our implementation of the FastCDC algorithm. * The hash of each chunk is calculated using the BLAKE3 hash function. * The length and hash of each chunk is appended to the file's chunk list. Prior to this change, when the user mounted a workstation directory on a client, the asset streaming server pushed an intermediate manifest to the gamelet as soon as step 1 was completed. At this point, the FUSE client started serving the virtual file system and was ready to answer `stat` and `readdir` calls. In case the FUSE client received any call that required file contents, such as `read`, it would block the caller until the server completed step 2 above and pushed the final manifest to the client. This works well for large directories (> 100GB) with a reasonable number of files (< 100k). But when dealing with millions of tiny files, creating the full manifest can take several minutes. With this change, we introduce dynamic manifest updates. When the FUSE layer receives an `open` or `readdir` request for a file or directory that is incomplete, it sends an RPC to the workstation about what information is missing from the manifest. The workstation identifies the corresponding file chunker or directory scanner tasks and moves them to the front of the queue. As soon as the task is completed, the workstation pushes an updated intermediate manifest to the client which now includes the information to serve the FUSE request. The queued FUSE request is resumed and returns the result to the caller. While this does not reduce the required time to build the final manifest, it splits up the work into smaller tasks. This allows us to interrupt the current work and prioritize those tasks which are required to handle an incoming request from the client. While this still takes a round-trip to the workstation plus the processing time for the task, an updated manifest is received within a few seconds, which is much better than blocking for several minutes. This latency is only visible when serving data while the manifest is still being created. The situation improves as the manifest creation on the workstation progresses. As soon as the final manifest is pushed, all metadata can be served directly without having to wait for pending tasks.
729 lines
28 KiB
C++
729 lines
28 KiB
C++
// Copyright 2022 Google LLC
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
#include "manifest/manifest_updater.h"
|
|
|
|
#include "absl/strings/match.h"
|
|
#include "common/path.h"
|
|
#include "common/status_test_macros.h"
|
|
#include "common/test_main.h"
|
|
#include "data_store/mem_data_store.h"
|
|
#include "fastcdc/fastcdc.h"
|
|
#include "gtest/gtest.h"
|
|
#include "manifest/file_chunk_map.h"
|
|
#include "manifest/manifest_builder.h"
|
|
#include "manifest/manifest_iterator.h"
|
|
#include "manifest/manifest_test_base.h"
|
|
|
|
namespace cdc_ft {
|
|
|
|
void PrintTo(const AssetInfo& ai, std::ostream* o) {
|
|
*o << "path=" << ai.path << ", type=" << ai.type << ", mtime=" << ai.mtime
|
|
<< ", size=" << ai.size;
|
|
}
|
|
|
|
namespace {
|
|
|
|
constexpr uint64_t kFileSizeA = 8; // a.txt
|
|
constexpr uint64_t kFileSizeB = 32; // subdir/b.txt
|
|
constexpr uint64_t kFileSizeC = 1; // subdir/c.txt
|
|
constexpr uint64_t kFileSizeD = 1; // subdir/d.txt
|
|
constexpr uint64_t kTotalFileSize =
|
|
kFileSizeA + kFileSizeB + kFileSizeC + kFileSizeD;
|
|
|
|
class ManifestUpdaterTest : public ManifestTestBase {
|
|
public:
|
|
ManifestUpdaterTest()
|
|
: ManifestTestBase(GetTestDataDir("manifest_updater")) {}
|
|
|
|
void SetUp() override {
|
|
path::CreateDirRec(empty_dir_).IgnoreError();
|
|
cfg_.num_threads = 1;
|
|
}
|
|
|
|
void TearDown() override { path::RemoveDirRec(empty_dir_).IgnoreError(); }
|
|
|
|
protected:
|
|
std::string empty_dir_ = path::Join(path::GetTempDir(), "empty");
|
|
};
|
|
|
|
// Runs UpdateAll() on an empty dir.
|
|
TEST_F(ManifestUpdaterTest, UpdateAll_EmptySrcDirectory) {
|
|
cfg_.src_dir = empty_dir_;
|
|
ManifestUpdater updater(&data_store_, cfg_);
|
|
EXPECT_OK(updater.UpdateAll(&file_chunks_));
|
|
|
|
UpdaterStats stats = updater.Stats();
|
|
EXPECT_EQ(stats.total_assets_added_or_updated, 0);
|
|
EXPECT_EQ(stats.total_files_added_or_updated, 0);
|
|
EXPECT_EQ(stats.total_files_failed, 0);
|
|
EXPECT_EQ(stats.total_assets_deleted, 0);
|
|
EXPECT_EQ(stats.total_chunks, 0);
|
|
EXPECT_EQ(stats.total_processed_bytes, 0);
|
|
|
|
// Store should contain a chunk for the manifest id and one for the manifest.
|
|
EXPECT_EQ(data_store_.Chunks().size(), 2);
|
|
ASSERT_NO_FATAL_FAILURE(ExpectManifestEquals({}, updater.ManifestId()));
|
|
}
|
|
|
|
// Runs UpdateAll() on a non-empty dir.
|
|
TEST_F(ManifestUpdaterTest, UpdateAll_NonEmptySrcDirectory) {
|
|
// Contains a.txt and subdir/b.txt.
|
|
cfg_.src_dir = path::Join(base_dir_, "non_empty");
|
|
ManifestUpdater updater(&data_store_, cfg_);
|
|
EXPECT_OK(updater.UpdateAll(&file_chunks_));
|
|
|
|
const UpdaterStats& stats = updater.Stats();
|
|
EXPECT_EQ(stats.total_assets_added_or_updated, 5);
|
|
EXPECT_EQ(stats.total_files_added_or_updated, 4);
|
|
EXPECT_EQ(stats.total_files_failed, 0);
|
|
EXPECT_EQ(stats.total_assets_deleted, 0);
|
|
EXPECT_EQ(stats.total_chunks, 4);
|
|
EXPECT_EQ(stats.total_processed_bytes, kTotalFileSize);
|
|
|
|
// Store should contain a chunk for the manifest id and one for the manifest.
|
|
EXPECT_EQ(data_store_.Chunks().size(), 2);
|
|
ASSERT_NO_FATAL_FAILURE(ExpectManifestEquals(
|
|
{"a.txt", "subdir", "subdir/b.txt", "subdir/c.txt", "subdir/d.txt"},
|
|
updater.ManifestId()));
|
|
}
|
|
|
|
// Runs UpdateAll() with existing manifest that misses a file.
|
|
TEST_F(ManifestUpdaterTest, UpdateAll_AddFileIncremental) {
|
|
// Create a manifest with "subdir/b.txt" missing.
|
|
cfg_.src_dir = path::Join(base_dir_, "non_empty");
|
|
ManifestUpdater updater(&data_store_, cfg_);
|
|
EXPECT_OK(updater.UpdateAll(&file_chunks_));
|
|
EXPECT_OK(updater.Update(
|
|
MakeDeleteOps({"subdir/b.txt", "subdir/c.txt", "subdir/d.txt"}),
|
|
&file_chunks_, nullptr));
|
|
ASSERT_NO_FATAL_FAILURE(
|
|
ExpectManifestEquals({"a.txt", "subdir"}, updater.ManifestId()));
|
|
|
|
// UpdateAll() should compute the proper diff from {"a.txt", "subdir"} to
|
|
// {"a.txt", "subdir", "subdir/b.txt", "subdir/c.txt", "subdir/d.txt"} and
|
|
// only add/update one file.
|
|
EXPECT_OK(updater.UpdateAll(&file_chunks_));
|
|
|
|
const UpdaterStats& stats = updater.Stats();
|
|
EXPECT_EQ(stats.total_assets_added_or_updated, 3);
|
|
EXPECT_EQ(stats.total_files_added_or_updated, 3);
|
|
EXPECT_EQ(stats.total_files_failed, 0);
|
|
EXPECT_EQ(stats.total_assets_deleted, 0);
|
|
EXPECT_EQ(stats.total_chunks, 3);
|
|
EXPECT_EQ(stats.total_processed_bytes, kFileSizeB + kFileSizeC + kFileSizeD);
|
|
ASSERT_NO_FATAL_FAILURE(ExpectManifestEquals(
|
|
{"a.txt", "subdir", "subdir/b.txt", "subdir/c.txt", "subdir/d.txt"},
|
|
updater.ManifestId()));
|
|
}
|
|
|
|
// Runs UpdateAll() with existing manifest that has an excessive file.
|
|
TEST_F(ManifestUpdaterTest, UpdateAll_DeleteFileIncremental) {
|
|
cfg_.src_dir = path::Join(base_dir_, "non_empty");
|
|
ManifestUpdater updater(&data_store_, cfg_);
|
|
EXPECT_OK(updater.UpdateAll(&file_chunks_));
|
|
|
|
// Smuggle c.txt into the manifest.
|
|
CdcParamsProto params;
|
|
params.set_min_chunk_size(cfg_.min_chunk_size);
|
|
params.set_avg_chunk_size(cfg_.avg_chunk_size);
|
|
params.set_max_chunk_size(cfg_.max_chunk_size);
|
|
ManifestBuilder mb(params, &data_store_);
|
|
EXPECT_OK(mb.LoadManifest(updater.ManifestId()));
|
|
EXPECT_OK(mb.GetOrCreateAsset("c.txt", AssetProto::FILE));
|
|
EXPECT_OK(mb.Flush());
|
|
std::string id_str = mb.ManifestId().SerializeAsString();
|
|
EXPECT_OK(data_store_.Put(manifest_store_id_, id_str.data(), id_str.size()));
|
|
|
|
// UpdateAll() should compute the proper diff from
|
|
// {"a.txt", "c.txt", "subdir", "subdir/b.txt"} to
|
|
// {"a.txt", "subdir", "subdir/b.txt", "subdir/c.txt", "subdir/d.txt"} and
|
|
// only delete one file.
|
|
EXPECT_OK(updater.UpdateAll(&file_chunks_));
|
|
|
|
const UpdaterStats& stats = updater.Stats();
|
|
EXPECT_EQ(stats.total_assets_added_or_updated, 0);
|
|
EXPECT_EQ(stats.total_files_added_or_updated, 0);
|
|
EXPECT_EQ(stats.total_files_failed, 0);
|
|
EXPECT_EQ(stats.total_assets_deleted, 1);
|
|
EXPECT_EQ(stats.total_chunks, 0);
|
|
EXPECT_EQ(stats.total_processed_bytes, 0);
|
|
ASSERT_NO_FATAL_FAILURE(ExpectManifestEquals(
|
|
{"a.txt", "subdir", "subdir/b.txt", "subdir/c.txt", "subdir/d.txt"},
|
|
updater.ManifestId()));
|
|
}
|
|
|
|
// UpdateAll() removes unreferenced manifest chunks.
|
|
TEST_F(ManifestUpdaterTest, UpdateAll_PrunesUnreferencedChunks) {
|
|
// Reduce chunk sizes to produce a bunch of indirect lists.
|
|
cfg_.min_chunk_size = 8;
|
|
cfg_.avg_chunk_size = 16;
|
|
cfg_.max_chunk_size = 32;
|
|
|
|
cfg_.src_dir = path::Join(base_dir_, "non_empty");
|
|
ManifestUpdater updater(&data_store_, cfg_);
|
|
EXPECT_OK(updater.Update(MakeUpdateOps({"a.txt"}), &file_chunks_, nullptr));
|
|
// 1 for manifest id, 1 for manifest, 1 indirect assets.
|
|
EXPECT_EQ(data_store_.Chunks().size(), 3);
|
|
|
|
EXPECT_OK(updater.Update(
|
|
MakeUpdateOps({"subdir/b.txt", "subdir/c.txt", "subdir/d.txt"}),
|
|
&file_chunks_, nullptr));
|
|
// 1 for manifest id, 1 for manifest, 5 indirect assets.
|
|
// 2 additional chunks from the first Update() that are now unreferenced.
|
|
// -1, because the indirect asset for "a.txt" is deduplicated
|
|
EXPECT_EQ(data_store_.Chunks().size(), 8)
|
|
<< "Manifest: " << ContentId::ToHexString(updater.ManifestId())
|
|
<< std::endl
|
|
<< DumpDataStoreProtos();
|
|
|
|
EXPECT_OK(updater.UpdateAll(&file_chunks_));
|
|
EXPECT_OK(updater.UpdateAll(&file_chunks_));
|
|
// 1 for manifest id, 1 for manifest, 5 indirect assets.
|
|
// Pruning has removed the 2 unreferenced ones.
|
|
EXPECT_EQ(data_store_.Chunks().size(), 7)
|
|
<< "Manifest: " << ContentId::ToHexString(updater.ManifestId())
|
|
<< std::endl
|
|
<< DumpDataStoreProtos();
|
|
}
|
|
|
|
// UpdateAll() recovers if there are missing referenced manifest chunks.
|
|
TEST_F(ManifestUpdaterTest, UpdateAll_RecoversFromMissingChunks) {
|
|
// Reduce chunk sizes to produce a bunch of indirect lists.
|
|
cfg_.min_chunk_size = 8;
|
|
cfg_.avg_chunk_size = 16;
|
|
cfg_.max_chunk_size = 32;
|
|
|
|
cfg_.src_dir = path::Join(base_dir_, "non_empty");
|
|
ManifestUpdater updater(&data_store_, cfg_);
|
|
EXPECT_OK(updater.Update(MakeUpdateOps({"a.txt"}), &file_chunks_, nullptr));
|
|
// 1 for manifest id, 1 for manifest, 1 indirect assets.
|
|
EXPECT_EQ(data_store_.Chunks().size(), 3)
|
|
<< "Manifest: " << ContentId::ToHexString(updater.ManifestId())
|
|
<< std::endl
|
|
<< DumpDataStoreProtos();
|
|
|
|
// Remove one of the indirect chunks list.
|
|
for (const auto& [id, _] : data_store_.Chunks()) {
|
|
if (id != ManifestUpdater::GetManifestStoreId() &&
|
|
id != updater.ManifestId()) {
|
|
data_store_.Chunks().erase(id);
|
|
break;
|
|
}
|
|
}
|
|
|
|
EXPECT_OK(updater.UpdateAll(&file_chunks_));
|
|
// 1 for manifest id, 1 for manifest, 5 indirect assets.
|
|
// There would be 8 chunks without the removal above, see UpdateAll_Prune.
|
|
EXPECT_EQ(data_store_.Chunks().size(), 7)
|
|
<< "Manifest: " << ContentId::ToHexString(updater.ManifestId())
|
|
<< std::endl
|
|
<< DumpDataStoreProtos();
|
|
}
|
|
|
|
// Verifies that |file_chunks_| contains the expected chunks after UpdateAll().
|
|
TEST_F(ManifestUpdaterTest, UpdateAll_FileChunkMapFromScratch) {
|
|
// Reduce chunk sizes to produce a bunch of indirect lists.
|
|
cfg_.min_chunk_size = 8;
|
|
cfg_.avg_chunk_size = 16;
|
|
cfg_.max_chunk_size = 32;
|
|
|
|
cfg_.src_dir = path::Join(base_dir_, "non_empty");
|
|
ManifestUpdater updater(&data_store_, cfg_);
|
|
EXPECT_OK(updater.UpdateAll(&file_chunks_));
|
|
|
|
ValidateChunkLookup("a.txt", true);
|
|
ValidateChunkLookup("subdir/b.txt", true);
|
|
ValidateChunkLookup("subdir/c.txt", true);
|
|
ValidateChunkLookup("subdir/d.txt", true);
|
|
}
|
|
|
|
// Verifies that |file_chunks_| contains the expected chunks after UpdateAll().
|
|
TEST_F(ManifestUpdaterTest, UpdateAll_FileChunkMapAfterUpdate) {
|
|
// Reduce chunk sizes to produce a bunch of indirect lists.
|
|
cfg_.min_chunk_size = 8;
|
|
cfg_.avg_chunk_size = 16;
|
|
cfg_.max_chunk_size = 32;
|
|
|
|
cfg_.src_dir = path::Join(base_dir_, "non_empty");
|
|
ManifestUpdater updater(&data_store_, cfg_);
|
|
ASSERT_OK(updater.UpdateAll(&file_chunks_));
|
|
// The file chunks will be populated again by UpdateAll().
|
|
file_chunks_.Clear();
|
|
EXPECT_OK(updater.UpdateAll(&file_chunks_));
|
|
|
|
ValidateChunkLookup("a.txt", true);
|
|
ValidateChunkLookup("subdir/b.txt", true);
|
|
ValidateChunkLookup("subdir/c.txt", true);
|
|
ValidateChunkLookup("subdir/d.txt", true);
|
|
}
|
|
|
|
// Verifies that the intermediate manifest contains the expected files.
|
|
TEST_F(ManifestUpdaterTest, UpdateAll_PushIntermediateManifest) {
|
|
ContentIdProto intermediate_id;
|
|
auto push_manifest = [&intermediate_id](const ContentIdProto& manifest_id) {
|
|
// Catch the first (= intermediate) manifest.
|
|
if (intermediate_id == ContentIdProto()) {
|
|
intermediate_id = manifest_id;
|
|
}
|
|
};
|
|
|
|
// Contains a.txt and subdir/b.txt.
|
|
cfg_.src_dir = path::Join(base_dir_, "non_empty");
|
|
ManifestUpdater updater(&data_store_, cfg_);
|
|
EXPECT_OK(updater.UpdateAll(&file_chunks_, push_manifest));
|
|
|
|
// Double check that the files in the final manifest are no longer in
|
|
// progress.
|
|
EXPECT_FALSE(InProgress(updater.ManifestId(), "a.txt"));
|
|
EXPECT_FALSE(InProgress(updater.ManifestId(), "subdir/b.txt"));
|
|
EXPECT_FALSE(InProgress(updater.ManifestId(), "subdir/c.txt"));
|
|
EXPECT_FALSE(InProgress(updater.ManifestId(), "subdir/d.txt"));
|
|
|
|
// Verify that the intermediate manifest is there, but it is empty.
|
|
std::string ser_id = intermediate_id.SerializeAsString();
|
|
EXPECT_OK(data_store_.Put(manifest_store_id_, ser_id.data(), ser_id.size()));
|
|
ASSERT_NO_FATAL_FAILURE(ExpectManifestEquals({}, intermediate_id));
|
|
// The root directory of the intermediate manifest is in progress.
|
|
EXPECT_TRUE(InProgress(intermediate_id, ""));
|
|
}
|
|
|
|
// Runs Update() with a single file to be added.
|
|
TEST_F(ManifestUpdaterTest, Update_AddFile) {
|
|
cfg_.src_dir = path::Join(base_dir_, "non_empty");
|
|
ManifestUpdater updater(&data_store_, cfg_);
|
|
EXPECT_OK(updater.Update(MakeUpdateOps({"a.txt"}), &file_chunks_, nullptr));
|
|
|
|
const UpdaterStats& stats = updater.Stats();
|
|
EXPECT_EQ(stats.total_assets_added_or_updated, 1);
|
|
EXPECT_EQ(stats.total_files_added_or_updated, 1);
|
|
EXPECT_EQ(stats.total_files_failed, 0);
|
|
EXPECT_EQ(stats.total_assets_deleted, 0);
|
|
EXPECT_EQ(stats.total_chunks, 1);
|
|
EXPECT_EQ(stats.total_processed_bytes, kFileSizeA);
|
|
ASSERT_NO_FATAL_FAILURE(
|
|
ExpectManifestEquals({"a.txt"}, updater.ManifestId()));
|
|
}
|
|
|
|
// Runs Update() with a single file to be added. The file is in a dir that is
|
|
// not contained in the manifest yet, so the dir will get auto-created.
|
|
TEST_F(ManifestUpdaterTest, Update_AddFileAutoCreateSubdir) {
|
|
cfg_.src_dir = path::Join(base_dir_, "non_empty");
|
|
ManifestUpdater updater(&data_store_, cfg_);
|
|
EXPECT_OK(
|
|
updater.Update(MakeUpdateOps({"subdir/b.txt"}), &file_chunks_, nullptr));
|
|
|
|
const UpdaterStats& stats = updater.Stats();
|
|
EXPECT_EQ(stats.total_assets_added_or_updated, 1);
|
|
EXPECT_EQ(stats.total_files_added_or_updated, 1);
|
|
EXPECT_EQ(stats.total_files_failed, 0);
|
|
EXPECT_EQ(stats.total_assets_deleted, 0);
|
|
EXPECT_EQ(stats.total_chunks, 1);
|
|
EXPECT_EQ(stats.total_processed_bytes, kFileSizeB);
|
|
|
|
// Note: The manifest does NOT contain the proper "subdir" asset now. Since it
|
|
// was auto-created because of "subdir/b.txt", it does not have the
|
|
// proper file time.
|
|
std::vector<AssetInfoForTest> manifest_ais =
|
|
GetAllManifestAssets(updater.ManifestId());
|
|
std::vector<AssetInfoForTest> expected_ais =
|
|
MakeAssetInfos({"subdir", "subdir/b.txt"});
|
|
ExpectAssetInfosEqual(manifest_ais, expected_ais, false);
|
|
manifest_ais[0].info.mtime = expected_ais[0].info.mtime;
|
|
ExpectAssetInfosEqual(manifest_ais, expected_ais, true);
|
|
}
|
|
|
|
// Calls Update() with a single file to be deleted.
|
|
TEST_F(ManifestUpdaterTest, Update_DeleteFiles) {
|
|
cfg_.src_dir = path::Join(base_dir_, "non_empty");
|
|
ManifestUpdater updater(&data_store_, cfg_);
|
|
EXPECT_OK(updater.UpdateAll(&file_chunks_));
|
|
EXPECT_OK(updater.Update(MakeDeleteOps({"a.txt"}), &file_chunks_, nullptr));
|
|
|
|
const UpdaterStats& stats = updater.Stats();
|
|
EXPECT_EQ(stats.total_assets_added_or_updated, 0);
|
|
EXPECT_EQ(stats.total_files_added_or_updated, 0);
|
|
EXPECT_EQ(stats.total_files_failed, 0);
|
|
EXPECT_EQ(stats.total_assets_deleted, 1);
|
|
EXPECT_EQ(stats.total_chunks, 0);
|
|
EXPECT_EQ(stats.total_processed_bytes, 0);
|
|
ASSERT_NO_FATAL_FAILURE(ExpectManifestEquals(
|
|
{"subdir", "subdir/b.txt", "subdir/c.txt", "subdir/d.txt"},
|
|
updater.ManifestId()));
|
|
|
|
// Delete another one in a subdirectory.
|
|
EXPECT_OK(
|
|
updater.Update(MakeDeleteOps({"subdir/b.txt"}), &file_chunks_, nullptr));
|
|
ASSERT_NO_FATAL_FAILURE(ExpectManifestEquals(
|
|
{"subdir", "subdir/c.txt", "subdir/d.txt"}, updater.ManifestId()));
|
|
}
|
|
|
|
// Calls Update() with a single dir to be deleted.
|
|
TEST_F(ManifestUpdaterTest, Update_DeleteDir) {
|
|
cfg_.src_dir = path::Join(base_dir_, "non_empty");
|
|
ManifestUpdater updater(&data_store_, cfg_);
|
|
EXPECT_OK(updater.UpdateAll(&file_chunks_));
|
|
EXPECT_OK(updater.Update(MakeDeleteOps({"subdir"}), &file_chunks_, nullptr));
|
|
|
|
const UpdaterStats& stats = updater.Stats();
|
|
EXPECT_EQ(stats.total_assets_added_or_updated, 0);
|
|
EXPECT_EQ(stats.total_files_added_or_updated, 0);
|
|
EXPECT_EQ(stats.total_files_failed, 0);
|
|
EXPECT_EQ(stats.total_assets_deleted, 1);
|
|
EXPECT_EQ(stats.total_chunks, 0);
|
|
EXPECT_EQ(stats.total_processed_bytes, 0);
|
|
ASSERT_NO_FATAL_FAILURE(
|
|
ExpectManifestEquals({"a.txt"}, updater.ManifestId()));
|
|
}
|
|
|
|
// Calls Update() with a non-existing asset to be deleted.
|
|
TEST_F(ManifestUpdaterTest, Update_DeleteNonExistingAsset) {
|
|
cfg_.src_dir = empty_dir_;
|
|
ManifestUpdater updater(&data_store_, cfg_);
|
|
// We need to craft AssetInfos for non-existing assets manually.
|
|
AssetInfo ai{"non_existing", AssetProto::DIRECTORY};
|
|
ManifestUpdater::OperationList ops{{Operator::kDelete, ai}};
|
|
EXPECT_OK(updater.Update(&ops, &file_chunks_, nullptr));
|
|
|
|
const UpdaterStats& stats = updater.Stats();
|
|
EXPECT_EQ(stats.total_assets_deleted, 1);
|
|
}
|
|
|
|
// Calls Update() with a non-existing file to be added.
|
|
TEST_F(ManifestUpdaterTest, Update_AddNonExistingFile) {
|
|
cfg_.src_dir = path::Join(base_dir_, "non_empty");
|
|
ManifestUpdater updater(&data_store_, cfg_);
|
|
|
|
// Note that Update() succeeds even through the "non_existing" file failed.
|
|
AssetInfo ai;
|
|
ai.path = "non_existing";
|
|
ManifestUpdater::OperationList ops{
|
|
{Operator::kAdd, ai}, {Operator::kAdd, MakeAssetInfo("a.txt").info}};
|
|
EXPECT_OK(updater.Update(&ops, &file_chunks_, nullptr));
|
|
|
|
const UpdaterStats& stats = updater.Stats();
|
|
EXPECT_EQ(stats.total_assets_added_or_updated, 2);
|
|
EXPECT_EQ(stats.total_files_added_or_updated, 1);
|
|
EXPECT_EQ(stats.total_files_failed, 1);
|
|
// "non_existing" and "a.txt" were still added, but the former is empty.
|
|
std::vector<AssetInfoForTest> manifest_ais =
|
|
GetAllManifestAssets(updater.ManifestId());
|
|
std::vector<AssetInfoForTest> expected_ais = {AssetInfoForTest{ai},
|
|
MakeAssetInfo("a.txt")};
|
|
ExpectAssetInfosEqual(manifest_ais, expected_ais);
|
|
}
|
|
|
|
// Verifies that the intermediate manifest contains the expected files.
|
|
TEST_F(ManifestUpdaterTest, Update_PushIntermediateManifest) {
|
|
// Create a manifest without a.txt.
|
|
cfg_.src_dir = path::Join(base_dir_, "non_empty");
|
|
ManifestUpdater updater(&data_store_, cfg_);
|
|
EXPECT_OK(updater.UpdateAll(&file_chunks_));
|
|
EXPECT_OK(updater.Update(
|
|
MakeDeleteOps({"subdir/b.txt", "subdir/c.txt", "subdir/d.txt"}),
|
|
&file_chunks_, nullptr));
|
|
|
|
// Add a.txt back and check intermediate manifest.
|
|
ContentIdProto intermediate_id;
|
|
auto push_manifest = [&intermediate_id](const ContentIdProto& manifest_id) {
|
|
// Catch the first (= intermediate) manifest.
|
|
if (intermediate_id == ContentIdProto()) {
|
|
intermediate_id = manifest_id;
|
|
}
|
|
};
|
|
EXPECT_OK(updater.Update(
|
|
MakeUpdateOps({"subdir/b.txt", "subdir/c.txt", "subdir/d.txt"}),
|
|
&file_chunks_, push_manifest));
|
|
EXPECT_GT(intermediate_id.blake3_sum_160().size(), 0);
|
|
|
|
// Only file a.txt is done in the intermediate manifest, all others are in
|
|
// progress.
|
|
EXPECT_FALSE(InProgress(intermediate_id, "a.txt"));
|
|
EXPECT_TRUE(InProgress(intermediate_id, "subdir/b.txt"));
|
|
EXPECT_TRUE(InProgress(intermediate_id, "subdir/c.txt"));
|
|
EXPECT_TRUE(InProgress(intermediate_id, "subdir/d.txt"));
|
|
}
|
|
|
|
// Verifies that |file_chunks_| contains the expected chunks after Update().
|
|
TEST_F(ManifestUpdaterTest, Update_FileChunkMap) {
|
|
// Reduce chunk sizes to produce a bunch of indirect lists.
|
|
cfg_.min_chunk_size = 8;
|
|
cfg_.avg_chunk_size = 16;
|
|
cfg_.max_chunk_size = 32;
|
|
|
|
cfg_.src_dir = path::Join(base_dir_, "non_empty");
|
|
ManifestUpdater updater(&data_store_, cfg_);
|
|
|
|
// Add a.txt.
|
|
EXPECT_OK(updater.Update(MakeUpdateOps({"a.txt"}), &file_chunks_, nullptr));
|
|
ValidateChunkLookup("a.txt", true);
|
|
ValidateChunkLookup("subdir/b.txt", false);
|
|
|
|
// Add subdir/b.txt.
|
|
EXPECT_OK(
|
|
updater.Update(MakeUpdateOps({"subdir/b.txt"}), &file_chunks_, nullptr));
|
|
ValidateChunkLookup("a.txt", true);
|
|
ValidateChunkLookup("subdir/b.txt", true);
|
|
|
|
// Remove a.txt.
|
|
EXPECT_OK(updater.Update(MakeDeleteOps({"a.txt"}), &file_chunks_, nullptr));
|
|
ValidateChunkLookup("a.txt", false);
|
|
ValidateChunkLookup("subdir/b.txt", true);
|
|
}
|
|
|
|
// Verifies that |file_chunks_| contains the expected chunks an intermediate
|
|
// update (and does not deadlock!).
|
|
TEST_F(ManifestUpdaterTest, Update_IntermediateFileChunkMap) {
|
|
cfg_.src_dir = path::Join(base_dir_, "non_empty");
|
|
ManifestUpdater updater(&data_store_, cfg_);
|
|
|
|
// Add a.txt.
|
|
EXPECT_OK(updater.Update(MakeUpdateOps({"a.txt"}), &file_chunks_, nullptr));
|
|
|
|
// Add subdir/b.txt and check intermediate lookups.
|
|
int count = 0;
|
|
auto push_manifest = [this, &count](const ContentIdProto&) {
|
|
++count;
|
|
ValidateChunkLookup("a.txt", true);
|
|
// The first (= intermediate) manifest does not have the chunks, the second
|
|
// (= final) does.
|
|
ValidateChunkLookup("subdir/b.txt", count > 1);
|
|
};
|
|
|
|
EXPECT_OK(updater.Update(MakeUpdateOps({"subdir/b.txt"}), &file_chunks_,
|
|
push_manifest));
|
|
}
|
|
|
|
// A call to ManifestId() returns the manifest id!!!
|
|
TEST_F(ManifestUpdaterTest, ManifestId) {
|
|
cfg_.src_dir = empty_dir_;
|
|
ManifestUpdater updater(&data_store_, cfg_);
|
|
EXPECT_OK(updater.UpdateAll(&file_chunks_));
|
|
|
|
ContentIdProto manifest_id;
|
|
EXPECT_OK(data_store_.GetProto(manifest_store_id_, &manifest_id));
|
|
EXPECT_EQ(updater.ManifestId(), manifest_id);
|
|
}
|
|
|
|
TEST_F(ManifestUpdaterTest, VerifyPermissions) {
|
|
cfg_.src_dir = path::Join(base_dir_, "non_empty");
|
|
ManifestUpdater updater(&data_store_, cfg_);
|
|
|
|
EXPECT_OK(updater.UpdateAll(&file_chunks_));
|
|
ManifestIterator manifest_iter(&data_store_);
|
|
EXPECT_OK(manifest_iter.Open(updater.ManifestId()));
|
|
const AssetProto* entry;
|
|
while ((entry = manifest_iter.NextEntry()) != nullptr) {
|
|
switch (entry->type()) {
|
|
case AssetProto::FILE:
|
|
EXPECT_EQ(entry->permissions(), ManifestBuilder::kDefaultFilePerms);
|
|
break;
|
|
case AssetProto::DIRECTORY:
|
|
EXPECT_EQ(entry->permissions(), ManifestBuilder::kDefaultDirPerms);
|
|
break;
|
|
case AssetProto::SYMLINK:
|
|
// Symlinks don't have their own permissions.
|
|
break;
|
|
default:
|
|
FAIL() << "Unhandled type: " << AssetProto::Type_Name(entry->type());
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
|
|
TEST_F(ManifestUpdaterTest, VerifyIntermediateFilesAreExecutable) {
|
|
cfg_.src_dir = path::Join(base_dir_, "non_empty");
|
|
ManifestUpdater updater(&data_store_, cfg_);
|
|
|
|
int count = 0;
|
|
auto push_intermediate_manifest = [this, &count](
|
|
const ContentIdProto& manifest_id) {
|
|
++count;
|
|
ManifestIterator manifest_iter(&data_store_);
|
|
EXPECT_OK(manifest_iter.Open(manifest_id));
|
|
const AssetProto* entry;
|
|
while ((entry = manifest_iter.NextEntry()) != nullptr) {
|
|
switch (entry->type()) {
|
|
case AssetProto::FILE:
|
|
if (count == 1) {
|
|
// While the manifest is in-progress, all files are set to be
|
|
// executable.
|
|
EXPECT_EQ(entry->permissions(), ManifestUpdater::kExecutablePerms);
|
|
} else {
|
|
EXPECT_EQ(entry->permissions(), ManifestBuilder::kDefaultFilePerms);
|
|
}
|
|
break;
|
|
case AssetProto::DIRECTORY:
|
|
EXPECT_EQ(entry->permissions(), ManifestBuilder::kDefaultDirPerms);
|
|
break;
|
|
default:
|
|
FAIL() << "Unhandled type: " << AssetProto::Type_Name(entry->type());
|
|
break;
|
|
}
|
|
}
|
|
};
|
|
|
|
// Add subdir/b.txt and verify the file permissions.
|
|
EXPECT_OK(updater.Update(MakeUpdateOps({"subdir/b.txt"}), &file_chunks_,
|
|
push_intermediate_manifest));
|
|
EXPECT_EQ(updater.Stats().total_files_added_or_updated, 1);
|
|
}
|
|
|
|
// Makes sure that executables are properly detected.
|
|
TEST_F(ManifestUpdaterTest, DetectExecutables) {
|
|
cfg_.src_dir = path::Join(base_dir_, "executables");
|
|
ManifestUpdater updater(&data_store_, cfg_);
|
|
EXPECT_OK(updater.UpdateAll(&file_chunks_));
|
|
|
|
ContentIdProto manifest_id;
|
|
EXPECT_OK(data_store_.GetProto(manifest_store_id_, &manifest_id));
|
|
|
|
ManifestIterator manifest_iter(&data_store_);
|
|
EXPECT_OK(manifest_iter.Open(manifest_id));
|
|
|
|
std::unordered_map<std::string, uint32_t> path_to_perms;
|
|
const AssetProto* entry;
|
|
while ((entry = manifest_iter.NextEntry()) != nullptr)
|
|
path_to_perms[entry->name()] = entry->permissions();
|
|
EXPECT_OK(manifest_iter.Status());
|
|
|
|
EXPECT_EQ(path_to_perms["game.elf"], ManifestUpdater::kExecutablePerms);
|
|
EXPECT_EQ(path_to_perms["win.exe"], ManifestUpdater::kExecutablePerms);
|
|
EXPECT_EQ(path_to_perms["script.sh"], ManifestUpdater::kExecutablePerms);
|
|
EXPECT_EQ(path_to_perms["normal.txt"], ManifestBuilder::kDefaultFilePerms);
|
|
}
|
|
|
|
TEST_F(ManifestUpdaterTest, UpdateAll_LargeIntermediateIndirectDirAssets) {
|
|
// Reduce chunk sizes to produce a bunch of indirect lists.
|
|
cfg_.min_chunk_size = 8;
|
|
cfg_.avg_chunk_size = 16;
|
|
cfg_.max_chunk_size = 32;
|
|
|
|
cfg_.src_dir = path::Join(base_dir_, "non_empty");
|
|
ManifestUpdater updater(&data_store_, cfg_);
|
|
|
|
// (internal): Run UpdateAll() with intermediate manifest push. The push
|
|
// causes a Flush() call to the manifest builder, which pushes some assets to
|
|
// indirect lists. This used to invalidate pointers and cause asserts to
|
|
// trigger.
|
|
EXPECT_OK(updater.UpdateAll(&file_chunks_, [](const ContentIdProto&) {}));
|
|
}
|
|
|
|
// Runs increamental UpdateAll() on an empty dir.
|
|
TEST_F(ManifestUpdaterTest, UpdateAll_EmptySrcDirectory_Incremental) {
|
|
cfg_.src_dir = empty_dir_;
|
|
ManifestUpdater updater(&data_store_, cfg_);
|
|
EXPECT_OK(updater.UpdateAll(&file_chunks_));
|
|
|
|
CdcParamsProto params;
|
|
params.set_min_chunk_size(cfg_.min_chunk_size);
|
|
params.set_avg_chunk_size(cfg_.avg_chunk_size);
|
|
params.set_max_chunk_size(cfg_.max_chunk_size);
|
|
ManifestBuilder mb(params, &data_store_);
|
|
EXPECT_OK(mb.LoadManifest(updater.ManifestId()));
|
|
EXPECT_OK(mb.GetOrCreateAsset("folder1", AssetProto::DIRECTORY));
|
|
EXPECT_OK(mb.DeleteAsset("folder1"));
|
|
}
|
|
|
|
TEST_F(ManifestUpdaterTest, UpdateAll_FileAsRootFails) {
|
|
cfg_.src_dir = path::Join(base_dir_, "non_empty", "a.txt");
|
|
ManifestUpdater updater(&data_store_, cfg_);
|
|
auto status = updater.UpdateAll(&file_chunks_);
|
|
EXPECT_NOT_OK(status);
|
|
EXPECT_TRUE(absl::IsFailedPrecondition(status)) << status.ToString();
|
|
}
|
|
|
|
TEST_F(ManifestUpdaterTest, UpdateAll_RootNotExistFails) {
|
|
cfg_.src_dir = path::Join(base_dir_, "non-existing");
|
|
ManifestUpdater updater(&data_store_, cfg_);
|
|
auto status = updater.UpdateAll(&file_chunks_);
|
|
EXPECT_NOT_OK(status);
|
|
EXPECT_TRUE(absl::IsNotFound(status)) << status.ToString();
|
|
}
|
|
|
|
// Runs UpdateAll() multiple times on an empty dir with no changes.
|
|
TEST_F(ManifestUpdaterTest, UpdateAll_EmptySrcDirectoryMultiTimesNoChange) {
|
|
cfg_.src_dir = empty_dir_;
|
|
ManifestUpdater updater(&data_store_, cfg_);
|
|
EXPECT_OK(updater.UpdateAll(&file_chunks_));
|
|
|
|
UpdaterStats stats = updater.Stats();
|
|
EXPECT_EQ(stats.total_assets_added_or_updated, 0);
|
|
EXPECT_EQ(stats.total_files_added_or_updated, 0);
|
|
EXPECT_EQ(stats.total_files_failed, 0);
|
|
EXPECT_EQ(stats.total_assets_deleted, 0);
|
|
EXPECT_EQ(stats.total_chunks, 0);
|
|
EXPECT_EQ(stats.total_processed_bytes, 0);
|
|
|
|
// Store should contain a chunk for the manifest id and one for the manifest.
|
|
EXPECT_EQ(data_store_.Chunks().size(), 2);
|
|
ASSERT_NO_FATAL_FAILURE(ExpectManifestEquals({}, updater.ManifestId()));
|
|
|
|
// No new changes should be done.
|
|
EXPECT_OK(updater.UpdateAll(&file_chunks_));
|
|
stats = updater.Stats();
|
|
EXPECT_EQ(stats.total_assets_added_or_updated, 0);
|
|
EXPECT_EQ(stats.total_files_added_or_updated, 0);
|
|
EXPECT_EQ(stats.total_files_failed, 0);
|
|
EXPECT_EQ(stats.total_assets_deleted, 0);
|
|
EXPECT_EQ(stats.total_chunks, 0);
|
|
EXPECT_EQ(stats.total_processed_bytes, 0);
|
|
}
|
|
|
|
// Runs UpdateAll() multiple times on a non-empty dir with no changes.
|
|
TEST_F(ManifestUpdaterTest, UpdateAll_NonEmptySrcDirectoryMultiTimesNoChange) {
|
|
// Contains a.txt and subdir/b.txt.
|
|
cfg_.src_dir = path::Join(base_dir_, "non_empty");
|
|
ManifestUpdater updater(&data_store_, cfg_);
|
|
EXPECT_OK(updater.UpdateAll(&file_chunks_));
|
|
|
|
UpdaterStats stats = updater.Stats();
|
|
EXPECT_EQ(stats.total_assets_added_or_updated, 5);
|
|
EXPECT_EQ(stats.total_files_added_or_updated, 4);
|
|
EXPECT_EQ(stats.total_files_failed, 0);
|
|
EXPECT_EQ(stats.total_assets_deleted, 0);
|
|
EXPECT_EQ(stats.total_chunks, 4);
|
|
EXPECT_EQ(stats.total_processed_bytes, kTotalFileSize);
|
|
|
|
// Store should contain a chunk for the manifest id and one for the manifest.
|
|
EXPECT_EQ(data_store_.Chunks().size(), 2);
|
|
ASSERT_NO_FATAL_FAILURE(ExpectManifestEquals(
|
|
{"a.txt", "subdir", "subdir/b.txt", "subdir/c.txt", "subdir/d.txt"},
|
|
updater.ManifestId()));
|
|
|
|
EXPECT_OK(updater.UpdateAll(&file_chunks_));
|
|
|
|
// No new changes should be done.
|
|
stats = updater.Stats();
|
|
EXPECT_EQ(stats.total_assets_added_or_updated, 0);
|
|
EXPECT_EQ(stats.total_files_added_or_updated, 0);
|
|
EXPECT_EQ(stats.total_files_failed, 0);
|
|
EXPECT_EQ(stats.total_assets_deleted, 0);
|
|
EXPECT_EQ(stats.total_chunks, 0);
|
|
EXPECT_EQ(stats.total_processed_bytes, 0);
|
|
}
|
|
|
|
TEST_F(ManifestUpdaterTest, IsValidDir) {
|
|
EXPECT_OK(ManifestUpdater::IsValidDir(path::Join(base_dir_, "non_empty")));
|
|
EXPECT_TRUE(absl::IsNotFound(
|
|
ManifestUpdater::IsValidDir(path::Join(base_dir_, "non-existing"))));
|
|
EXPECT_TRUE(absl::IsFailedPrecondition(ManifestUpdater::IsValidDir(
|
|
path::Join(base_dir_, "non_empty", "a.txt"))));
|
|
EXPECT_TRUE(
|
|
absl::IsFailedPrecondition(ManifestUpdater::IsValidDir("relative_dir")));
|
|
}
|
|
|
|
} // namespace
|
|
} // namespace cdc_ft
|