Publish Error Prone compatibility matrix on website (#938)

The new `website/generate-version-compatibility-overview.sh` script
tests all combinations, and stores the result in a Jekyll data file.

Resolves #724.

website/_config.yml

@@ -14,6 +14,7 @@ plugins:
 exclude:
   - Gemfile
   - Gemfile.lock
+  - generate-version-compatibility-overview.sh
   - README.md
   - vendor
 

website/_data/compatibility.yml

@@ -0,0 +1,82 @@
+# An overview of Error Prone Support releases, along with compatible Error
+# Prone releases. This data was generated by `generate-version-compatibility-overview.sh`.
+releases:
+  - version: 0.14.0
+    compatible:
+      - "2.24.0"
+      - "2.23.0"
+      - "2.22.0"
+  - version: 0.13.0
+    compatible:
+      - "2.22.0"
+      - "2.21.1"
+      - "2.21.0"
+      - "2.20.0"
+  - version: 0.12.0
+    compatible:
+      - "2.22.0"
+      - "2.21.1"
+      - "2.21.0"
+      - "2.20.0"
+  - version: 0.11.1
+    compatible:
+      - "2.19.1"
+      - "2.19.0"
+  - version: 0.11.0
+    compatible:
+      - "2.19.1"
+      - "2.19.0"
+  - version: 0.10.0
+    compatible:
+      - "2.18.0"
+  - version: 0.9.0
+    compatible:
+      - "2.18.0"
+  - version: 0.8.0
+    compatible:
+      - "2.18.0"
+  - version: 0.7.0
+    compatible:
+      - "2.17.0"
+  - version: 0.6.0
+    compatible:
+      - "2.16"
+  - version: 0.5.0
+    compatible:
+      - "2.16"
+  - version: 0.4.0
+    compatible:
+      - "2.16"
+      - "2.15.0"
+      - "2.14.0"
+      - "2.13.1"
+      - "2.13.0"
+      - "2.12.1"
+      - "2.12.0"
+  - version: 0.3.0
+    compatible:
+      - "2.16"
+      - "2.15.0"
+      - "2.14.0"
+      - "2.13.1"
+      - "2.13.0"
+      - "2.12.1"
+      - "2.12.0"
+  - version: 0.2.0
+    compatible:
+      - "2.16"
+      - "2.15.0"
+      - "2.14.0"
+      - "2.13.1"
+      - "2.13.0"
+      - "2.12.1"
+      - "2.12.0"
+  - version: 0.1.0
+    compatible:
+      - "2.16"
+      - "2.15.0"
+      - "2.14.0"
+      - "2.13.1"
+      - "2.13.0"
+      - "2.12.1"
+      - "2.12.0"

website/compatibility.md

@@ -0,0 +1,36 @@
+---
+layout: default
+title: Compatibility matrix
+nav_order: 2
+---
+
+# Compatibility matrix
+
+{% comment %}
+XXX: Once available on the default branch, include a link to the
+`generate-version-compatibility-overview.sh` script.
+{% endcomment %}
+
+Error Prone Support releases are generally compatible with only a limited
+number of Error Prone releases. The table below shows, for each Error Prone
+Support release, the Error Prone versions it is expected to be compatible with.
+Compatibility is determined by:
+1. Compiling and testing the Error Prone Support release source code against a
+   given Error Prone version. This validates source and behavioral
+   compatibility.[^1] [^2]
+2. Applying the released Refaster rules using a given Error Prone version. This
+   validates that the rules can be read by the targeted version of Error Prone,
+   proving that the serialization format is compatible.
+
+| Error Prone Support version | Compatible Error Prone versions |
+| --------------------------- | ------------------------------- |
+{% for release in site.data.compatibility.releases -%}
+| [{{ release.version }}](https://github.com/PicnicSupermarket/error-prone-support/releases/tag/v{{ release.version }}) | {%
+for version in release.compatible -%}
+[{{ version }}](https://github.com/google/error-prone/releases/tag/v{{ version }}){% unless forloop.last %}, {% endunless %}
+{%- endfor %} |
+{% endfor %}
+
+[^1]: Note that this [does not prove][source-binary-compat] that the Error Prone Support and Error Prone versions are _binary_ compatible. This limitation does not appear to be an issue in practice.
+[^2]: The approach taken here may yield false negatives, because a reported incompatibility may merely be due to a test API incompatibility.
+[source-binary-compat]: https://stackoverflow.com/questions/57871898/

website/generate-version-compatibility-overview.sh

@@ -0,0 +1,86 @@
+#!/usr/bin/env bash
+
+# Determines, for each Error Prone Support release, with which Error Prone
+# releases it is (very likely to be) compatible. The result is written to
+# `./_data/compatibility.yml`, which is a source for `./compatibility.md`.
+
+# This script relies on SDKMAN! to install compatible Maven versions.
+export SDKMAN_OFFLINE_MODE=false
+source "${HOME}/.sdkman/bin/sdkman-init.sh"
+
+set -e -u -o pipefail
+
+output_file="$(dirname "${0}")/_data/compatibility.yml"
+
+ep_versions=$(
+  git ls-remote \
+      --exit-code --refs --sort='-v:refname' \
+      https://github.com/google/error-prone.git \
+      'v*.*' \
+    | grep -oP '(?<=/v)[^/]+$'
+)
+
+work_dir="$(mktemp -d)"
+trap 'rm -rf -- "${work_dir=}"' INT TERM HUP EXIT
+build_log="${work_dir}/build.log"
+
+git clone -q [email protected]:PicnicSupermarket/error-prone-support.git "${work_dir}"
+pushd "${work_dir}" > /dev/null
+
+eps_versions="$(git tag --list --sort='-v:refname' 'v*.*.*')"
+
+# Check out the source of each Error Prone Support release, and try to build
+# and test it against each Error Prone release.
+for eps_version in ${eps_versions}; do
+  git checkout --force "${eps_version}" --
+
+  # Make sure to build with a compatible version of Maven.
+  mvn_version="$(grep -oP '(?<=<version.maven>)[^>]+(?=</version.maven>)' pom.xml)"
+  (set +u && echo n | sdk install maven "${mvn_version}")
+  sdk use maven "${mvn_version}"
+
+  # Remove any Error Prone flags used by this build that may not be compatible
+  # with the targeted version of Error Prone. Removal of these build flags does
+  # not influence the compatibility assessment.
+  sed -i -r 's,-XepAllSuggestionsAsWarnings|-Xep:\w+:\w+,,g' pom.xml
+
+  # Using each Error Prone release, attempt to build and test the source, while
+  # also applying the Maven Central-hosted Refaster rules. This determines
+  # source and behavioral (in)compatibility with Error Prone APIs, while also
+  # assessing whether the Refaster rules are deserialization-compatible.
+  for ep_version in ${ep_versions}; do
+    echo "Testing Error Prone Support ${eps_version} with Error Prone ${ep_version}..."
+    mvn clean test \
+        -Perror-prone \
+        -Derror-prone.patch-checks=Refaster \
+        -Ppatch \
+        -Pself-check \
+        -Dverification.skip \
+        -Dversion.error-prone-orig="${ep_version}" \
+      && echo "SUCCESS: { \"eps_version\": \"${eps_version}\", \"ep_version\": \"${ep_version}\" }" || true
+    # Undo any changes applied by Refaster.
+    git checkout -- '*.java'
+  done
+done | tee "${build_log}"
+
+popd
+
+# Regenerate the Jekyll compatibility data file by extracting the collected
+# data from the build log.
+cat > "${output_file}" << EOF
+# An overview of Error Prone Support releases, along with compatible Error
+# Prone releases. This data was generated by \`$(basename "${0}")\`.
+EOF
+grep -oP '(?<=SUCCESS: ).*' "${build_log}" \
+  | jq -s '.' \
+  | yq -r '
+      {
+        "releases":
+          group_by(.eps_version)
+            | map({
+                "version": .[0].eps_version | sub("^v", ""),
+                "compatible": map(.ep_version)
+              })
+      }
+    ' \
+  | tee -a "${output_file}"