From 09306171491b71a41d1715f8c2cb74a09b5971d2 Mon Sep 17 00:00:00 2001 From: Benjy Weinberger Date: Fri, 12 Aug 2022 12:57:57 -0400 Subject: [PATCH 1/5] Better guidance on how to set up CI with Pants. --- .github/workflows/pants.yaml | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pants.yaml b/.github/workflows/pants.yaml index 3d1f997..9dbe064 100644 --- a/.github/workflows/pants.yaml +++ b/.github/workflows/pants.yaml @@ -30,10 +30,24 @@ jobs: id: cache with: path: | - ~/.cache/pants/setup - ~/.cache/pants/lmdb_store - ~/.cache/pants/named_caches - key: ${{ runner.os }}- + ~/.cache/pants/setup # The initial bootstrapping of Pants + ~/.cache/pants/named_caches # Caches used by underlying tools like Pip and PEX. + # If you're not using a fine-grained remote caching service, such as the one provided by + # Toolchain(https://toolchain.com/), the lead sponsor of Pants, then you may also want to + # preserve the local Pants cache: + # ~/.cache/pants/lmdb_store + # Note that this is a very non-specific key, so the cache will include the + # setup for every prior version of Pants you've used, and a lot of old state of + # underlying tools, and will need to be pruned from time to time. + # See https://www.pantsbuild.org/docs/using-pants-in-ci for examples of how to do so. + # + # Alternatively, you could use a key like `pants-setup-${{ runner.os }}-${{ hashFiles('pants.toml') }} + # which will invalidate when the pants version changes, but will also invalidate on any other + # config change, which may be too aggressive. + # + # If you cache the lmdb_store directory, that will grow quickly and also need to be pruned. + # A Pants-aware, fine-grained remote cache service avoids these problems. + key: pants-setup-${{ runner.os }} - name: Setup Python ${{ matrix.python-version }} uses: actions/setup-python@v1 with: From 49aa3ea1ad1d8f2ba121b4638fec8343f9cdddac Mon Sep 17 00:00:00 2001 From: Benjy Weinberger Date: Fri, 12 Aug 2022 15:00:05 -0400 Subject: [PATCH 2/5] Link to pants docs instead --- .github/workflows/pants.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pants.yaml b/.github/workflows/pants.yaml index 9dbe064..1b7e4c4 100644 --- a/.github/workflows/pants.yaml +++ b/.github/workflows/pants.yaml @@ -32,9 +32,9 @@ jobs: path: | ~/.cache/pants/setup # The initial bootstrapping of Pants ~/.cache/pants/named_caches # Caches used by underlying tools like Pip and PEX. - # If you're not using a fine-grained remote caching service, such as the one provided by - # Toolchain(https://toolchain.com/), the lead sponsor of Pants, then you may also want to - # preserve the local Pants cache: + # If you're not using a fine-grained remote caching service (see + # https://www.pantsbuild.org/docs/remote-caching), then you may also want to preserve + # the local Pants cache: # ~/.cache/pants/lmdb_store # Note that this is a very non-specific key, so the cache will include the # setup for every prior version of Pants you've used, and a lot of old state of From 31a7be8b2fa26c8fb6150fe1da96728b8e1902d1 Mon Sep 17 00:00:00 2001 From: Benjy Weinberger Date: Fri, 12 Aug 2022 17:15:13 -0400 Subject: [PATCH 3/5] Get this right --- .github/workflows/pants.yaml | 48 ++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 19 deletions(-) diff --git a/.github/workflows/pants.yaml b/.github/workflows/pants.yaml index 1b7e4c4..7c0d94d 100644 --- a/.github/workflows/pants.yaml +++ b/.github/workflows/pants.yaml @@ -27,27 +27,37 @@ jobs: steps: - uses: actions/checkout@v2 - uses: actions/cache@v2 - id: cache + id: cache_pants_setup with: path: | - ~/.cache/pants/setup # The initial bootstrapping of Pants - ~/.cache/pants/named_caches # Caches used by underlying tools like Pip and PEX. - # If you're not using a fine-grained remote caching service (see - # https://www.pantsbuild.org/docs/remote-caching), then you may also want to preserve - # the local Pants cache: - # ~/.cache/pants/lmdb_store - # Note that this is a very non-specific key, so the cache will include the - # setup for every prior version of Pants you've used, and a lot of old state of - # underlying tools, and will need to be pruned from time to time. - # See https://www.pantsbuild.org/docs/using-pants-in-ci for examples of how to do so. - # - # Alternatively, you could use a key like `pants-setup-${{ runner.os }}-${{ hashFiles('pants.toml') }} - # which will invalidate when the pants version changes, but will also invalidate on any other - # config change, which may be too aggressive. - # - # If you cache the lmdb_store directory, that will grow quickly and also need to be pruned. - # A Pants-aware, fine-grained remote cache service avoids these problems. - key: pants-setup-${{ runner.os }} + ~/.cache/pants/setup + key: pants-setup-${{ runner.os }}-${{ hashFiles('pants.toml') }} + restore-keys: | + pants-setup-${{ runner.os }}-${{ hashFiles('pants.toml') }} + pants-setup-${{ runner.os }}- + - uses: actions/cache@v2 + id: cached_named_caches + with: + path: | + ~/.cache/pants/named_caches + # The Python backend uses named_caches for Pip/PEX state, + # so it is appropriate to invalidate on requirements.txt changes. + key: named-caches-${{ runner.os }}-${{ hashFiles('pants.toml') }}-${{ hashFiles('requirements.txt') }} + restore-keys: | + pants-setup-${{ runner.os }}-${{ hashFiles('pants.toml') }}-${{ hashFiles('requirements.txt') }} + pants-setup-${{ runner.os }}-${{ hashFiles('pants.toml') }}- + pants-setup-${{ runner.os }}- + # If you're not using a fine-grained remote caching service (see https://www.pantsbuild.org/docs/remote-caching), + # then you may also want to preserve the local Pants cache (lmdb_store). However this must invalidate for + # changes to any file that can affect the build, so may not be practical in larger repos. + # A remote cache service integrates with Pants's fine-grained invalidation and avoids these problems. + - uses: actions/cache@v2 + id: cache_lmdb_store + with: + path: | + ~/.cache/pants/lmdb_store + key: pants-setup-${{ runner.os }}-${{ hashFiles('**/*') }} + restore-keys: pants-setup-${{ runner.os }}- - name: Setup Python ${{ matrix.python-version }} uses: actions/setup-python@v1 with: From 1b4dc2e950001fb6c0d7937b3dab01a00d7d99c3 Mon Sep 17 00:00:00 2001 From: Benjy Weinberger Date: Sat, 13 Aug 2022 09:42:34 -0400 Subject: [PATCH 4/5] Improvements --- .github/workflows/pants.yaml | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pants.yaml b/.github/workflows/pants.yaml index 7c0d94d..78abed0 100644 --- a/.github/workflows/pants.yaml +++ b/.github/workflows/pants.yaml @@ -26,15 +26,18 @@ jobs: python-version: [3.7] steps: - uses: actions/checkout@v2 + - name: Get Pants version + id: pants_version + run: | + # Capture the "pants_version = " line from config. + PANTS_VERSION=$(grep -E '^pants_version\s*=' pants.toml) + echo "::set-output name=pants_version::$PANTS_VERSION" - uses: actions/cache@v2 id: cache_pants_setup with: path: | ~/.cache/pants/setup - key: pants-setup-${{ runner.os }}-${{ hashFiles('pants.toml') }} - restore-keys: | - pants-setup-${{ runner.os }}-${{ hashFiles('pants.toml') }} - pants-setup-${{ runner.os }}- + key: pants-setup-${{ steps.pants_version.outputs.pants_version }} - uses: actions/cache@v2 id: cached_named_caches with: @@ -45,6 +48,10 @@ jobs: key: named-caches-${{ runner.os }}-${{ hashFiles('pants.toml') }}-${{ hashFiles('requirements.txt') }} restore-keys: | pants-setup-${{ runner.os }}-${{ hashFiles('pants.toml') }}-${{ hashFiles('requirements.txt') }} + # Note that falling back to a restore key may give a useful partial result that will save time + # over completely clean state, but will cause the cache entry to grow without bound over time. + # See https://pants.readme.io/docs/using-pants-in-ci for tips on how to periodically clean it up. + # Alternatively you may want to avoid using restore keys. pants-setup-${{ runner.os }}-${{ hashFiles('pants.toml') }}- pants-setup-${{ runner.os }}- # If you're not using a fine-grained remote caching service (see https://www.pantsbuild.org/docs/remote-caching), @@ -57,6 +64,7 @@ jobs: path: | ~/.cache/pants/lmdb_store key: pants-setup-${{ runner.os }}-${{ hashFiles('**/*') }} + # Same caveat as above regarding the issues with restore keys. restore-keys: pants-setup-${{ runner.os }}- - name: Setup Python ${{ matrix.python-version }} uses: actions/setup-python@v1 From adfadf2c50f5deadf0e5d6ec1fc09fdca28a8705 Mon Sep 17 00:00:00 2001 From: Benjy Weinberger Date: Sat, 13 Aug 2022 09:44:27 -0400 Subject: [PATCH 5/5] Fix --- .github/workflows/pants.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pants.yaml b/.github/workflows/pants.yaml index 78abed0..8249954 100644 --- a/.github/workflows/pants.yaml +++ b/.github/workflows/pants.yaml @@ -46,12 +46,12 @@ jobs: # The Python backend uses named_caches for Pip/PEX state, # so it is appropriate to invalidate on requirements.txt changes. key: named-caches-${{ runner.os }}-${{ hashFiles('pants.toml') }}-${{ hashFiles('requirements.txt') }} + # Note that falling back to a restore key may give a useful partial result that will save time + # over completely clean state, but will cause the cache entry to grow without bound over time. + # See https://pants.readme.io/docs/using-pants-in-ci for tips on how to periodically clean it up. + # Alternatively you may want to avoid using restore keys. restore-keys: | pants-setup-${{ runner.os }}-${{ hashFiles('pants.toml') }}-${{ hashFiles('requirements.txt') }} - # Note that falling back to a restore key may give a useful partial result that will save time - # over completely clean state, but will cause the cache entry to grow without bound over time. - # See https://pants.readme.io/docs/using-pants-in-ci for tips on how to periodically clean it up. - # Alternatively you may want to avoid using restore keys. pants-setup-${{ runner.os }}-${{ hashFiles('pants.toml') }}- pants-setup-${{ runner.os }}- # If you're not using a fine-grained remote caching service (see https://www.pantsbuild.org/docs/remote-caching),