From c9e18b9e91ada400cf3d49be9b1bee7164301143 Mon Sep 17 00:00:00 2001 From: Lutz Justen Date: Mon, 5 Dec 2022 10:46:56 +0100 Subject: [PATCH] Reuse bazel cache folder between builds (#38) Uses a bazel --disk_cache to cache build outputs between builds. Bazel also has a local cache, e.g. in ~/.cache/bazel/_bazel_$USER/cache, but that one can't be used as it won't reuse data across checkouts. A disk cache is like a remote cache, except that it's on the local disk. Github first looks for a cache with the given exact key in the current branch, then in the main branch. If there's a cache hit, the cache isn't updated (they're read-only!). To prevent that caches become stale, they are timestamped using the current year and month, so that the cache is force-renewed every month. Bazel disk caches also just grow, so this technique prevents that the cache grows indefinitely, eventually causing cache trashing. --- .github/workflows/build_and_test.yml | 33 ++++++++++++++++++++++------ .github/workflows/create_release.yml | 30 +++++++++++++++++++++---- 2 files changed, 52 insertions(+), 11 deletions(-) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index b650fbe..96d19f0 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -23,17 +23,26 @@ jobs: - name: Initialize submodules run: git submodule update --init --recursive - - name: Build (fastbuild) + - name: Create timestamp run: | - bazel build --config=linux -- //... -//third_party/... + printf -v date '%(%Y-%m)T' -1 + echo "date=$date" >> $GITHUB_ENV + + - name: Restore build cache + uses: actions/cache@v3 + with: + path: bazel-cache + key: ${{ runner.os }}-bazel-cache-${{ env.date }} + + - name: Build (fastbuild) + run: bazel build --config=linux --disk_cache=bazel-cache -- //... -//third_party/... # Skip file_finder_test: The test works when file_finder_test is run # directly, but not through bazel test. The reason is, bazel test # creates symlinks of test files, but the finder ignores symlinks. # Also run tests sequentially since some tests write to a common tmp dir. - name: Test (fastbuild) - run: | - bazel test --config=linux --test_output=errors --local_test_jobs=1 -- //... -//third_party/... -//cdc_rsync_server:file_finder_test + run: bazel test --config=linux --disk_cache=bazel-cache --test_output=errors --local_test_jobs=1 -- //... -//third_party/... -//cdc_rsync_server:file_finder_test Build-And-Test-Windows: runs-on: windows-2019 @@ -43,16 +52,26 @@ jobs: - name: Initialize submodules run: git submodule update --init --recursive - - name: Build + - name: Create timestamp run: | - bazel build --config=windows //cdc_rsync //cdc_stream //tests_common //tests_cdc_stream //tests_cdc_rsync + $date = Get-Date -Format "yyyy-MM" + echo "date=$date" >> $GITHUB_ENV + + - name: Restore build cache + uses: actions/cache@v3 + with: + path: bazel-cache + key: ${{ runner.os }}-bazel-cache-${{ env.date }} + + - name: Build + run: bazel build --config=windows --disk_cache=bazel-cache //cdc_rsync //cdc_stream //tests_common //tests_cdc_stream //tests_cdc_rsync - name: Test run: | bazel-bin\tests_common\tests_common.exe bazel-bin\tests_cdc_stream\tests_cdc_stream.exe bazel-bin\tests_cdc_rsync\tests_cdc_rsync.exe - bazel test --config=windows --test_output=errors --local_test_jobs=1 ` + bazel test --config=windows --disk_cache=bazel-cache --test_output=errors --local_test_jobs=1 ` //cdc_fuse_fs/... ` //cdc_rsync/... ` //cdc_rsync/base/... ` diff --git a/.github/workflows/create_release.yml b/.github/workflows/create_release.yml index b1148f7..418c207 100644 --- a/.github/workflows/create_release.yml +++ b/.github/workflows/create_release.yml @@ -17,14 +17,25 @@ jobs: - name: Initialize submodules run: git submodule update --init --recursive + - name: Create timestamp + run: | + printf -v date '%(%Y-%m)T' -1 + echo "date=$date" >> $GITHUB_ENV + + - name: Restore build cache + uses: actions/cache@v3 + with: + path: bazel-cache + key: ${{ runner.os }}-bazel-cache-${{ env.date }} + - name: Build run: | - bazel build --config=linux --compilation_mode=opt --linkopt=-Wl,--strip-all --copt=-fdata-sections --copt=-ffunction-sections --linkopt=-Wl,--gc-sections \ + bazel build --config=linux --disk_cache=bazel-cache --compilation_mode=opt --linkopt=-Wl,--strip-all --copt=-fdata-sections --copt=-ffunction-sections --linkopt=-Wl,--gc-sections \ //cdc_fuse_fs //cdc_rsync_server - name: Test run: | - bazel test --config=linux --compilation_mode=opt --linkopt=-Wl,--strip-all --copt=-fdata-sections --copt=-ffunction-sections --linkopt=-Wl,--gc-sections \ + bazel test --config=linux --disk_cache=bazel-cache --compilation_mode=opt --linkopt=-Wl,--strip-all --copt=-fdata-sections --copt=-ffunction-sections --linkopt=-Wl,--gc-sections \ --test_output=errors --local_test_jobs=1 \ -- //... -//third_party/... -//cdc_rsync_server:file_finder_test @@ -50,9 +61,20 @@ jobs: - name: Initialize submodules run: git submodule update --init --recursive + - name: Create timestamp + run: | + $date = Get-Date -Format "yyyy-MM" + echo "date=$date" >> $GITHUB_ENV + + - name: Restore build cache + uses: actions/cache@v3 + with: + path: bazel-cache + key: ${{ runner.os }}-bazel-cache-${{ env.date }} + - name: Build run: | - bazel build --config=windows --compilation_mode=opt --copt=/GL ` + bazel build --config=windows --disk_cache=bazel-cache --compilation_mode=opt --copt=/GL ` //cdc_rsync //cdc_stream //tests_common //tests_cdc_stream //tests_cdc_rsync - name: Test @@ -60,7 +82,7 @@ jobs: bazel-bin\tests_common\tests_common.exe bazel-bin\tests_cdc_stream\tests_cdc_stream.exe bazel-bin\tests_cdc_rsync\tests_cdc_rsync.exe - bazel test --config=windows --compilation_mode=opt --copt=/GL --test_output=errors --local_test_jobs=1 ` + bazel test --config=windows --disk_cache=bazel-cache --compilation_mode=opt --copt=/GL --test_output=errors --local_test_jobs=1 ` //cdc_fuse_fs/... ` //cdc_rsync/... ` //cdc_rsync/base/... `