feat: unmanaged dependency check (#2223)

* feat: add a unmanaged dependency check * feat: add unmanaged dependency check * change parent * change source/target * add private constructor * throw exception if the private constructor is called * remove module * add main function * add action yaml * add ci to check dependencies * delete unused file * change action path * change working directory * install modules * parse version of shared-dependencies * upgrade checkout action * add error exit * add an unmanaged dependency * surpress mvn output * add slf4j * add echo to debug * checkout branch head * add an unit test * add an execution * change phase * use ClassPathBuilder * restore BOM * fix shared dependency version * change check's description * change version * run unit tests in ci * change error message * only run tests in unmanaged dependency check module * change install command * add a debug echo * change exec path * do not install check * change working dir * print mvn log * install modules * modify ut * add javadoc * use github.action_path * change pom path * change command sequence * exclude handwritten artifacts * retrieve latest shared dependencies using git tag * add a variable * debug * combine steps * use shared dependencies bom path * install pom in test * revert depdendency * install shared dependencies in action.yaml * add a tag for the check * refactor * restore public * use artifact instead of toString * change error message * change variable name * remove main method * Revert "remove main method" This reverts commit 38b3ee4. * change description
googleapis · Jan 3, 2024 · 3439691 · 3439691
1 parent c7de93e
commit 3439691
Show file tree

Hide file tree

Showing 12 changed files with 529 additions and 3 deletions.
diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
@@ -299,3 +299,25 @@ jobs:
       with:
         bom-path: gapic-generator-java-bom/pom.xml
 
+  unmanaged_dependency_check:
+    runs-on: ubuntu-22.04
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        ref: ${{ github.event.pull_request.head.sha }}
+    - uses: actions/setup-java@v3
+      with:
+        java-version: 11
+        distribution: temurin
+    - run: mvn -version
+    - name: Unit Tests
+      run: |
+        mvn test --batch-mode --no-transfer-progress
+      working-directory: java-shared-dependencies/unmanaged-dependency-check
+    - name: Install Maven modules
+      run: |
+        mvn install -B -ntp -DskipTests -Dclirr.skip -Dcheckstyle.skip
+    - name: Unmanaged dependency check
+      uses: ./java-shared-dependencies/unmanaged-dependency-check
+      with:
+        bom-path: gapic-generator-java-bom/pom.xml
diff --git a/.github/workflows/create_additional_release_tag.yaml b/.github/workflows/create_additional_release_tag.yaml
@@ -14,16 +14,14 @@ jobs:
       uses: actions/checkout@v3
       with:
         token: ${{ secrets.GITHUB_TOKEN }}
-
     - name: Set up Git
       run: |
         git config --local user.email "[email protected]"
         git config --local user.name "GitHub Action"
-
     - name: Fetch all tags
       run: git fetch --tags
-
     - name: Create additional tags
+      bash: shell
       run: |
         ARTIFACT_IDS=('google-cloud-shared-dependencies' 'api-common' 'gax')
         for ARTIFACT_ID in "${ARTIFACT_IDS[@]}"; do
@@ -36,3 +34,10 @@ jobs:
           git tag $TAG_NAME
           git push origin $TAG_NAME
         done
+        # Generate a tag for unmanaged dependencies check.
+        # Use fixed tag so that checks in handwritten libraries do not need to
+        # update the version.
+        CHECK_LATEST_TAG="unmanaged-dependencies-check-latest"
+        git tag ${CHECK_LATEST_TAG}
+        git push origin -f ${CHECK_LATEST_TAG}
+
diff --git a/java-shared-dependencies/unmanaged-dependency-check/action.yaml b/java-shared-dependencies/unmanaged-dependency-check/action.yaml
@@ -0,0 +1,43 @@
+  name: "Unmanaged dependency check"
+  description: "Checks whether there's a dependency that is not managed by java shared dependencies."
+  inputs:
+    bom-path:
+      description: "The relative path from the repository root to the pom.xml file"
+      required: true
+  runs:
+    using: "composite"
+    steps:
+    - uses: actions/setup-java@v3
+      with:
+        distribution: temurin
+        java-version: 11
+        cache: maven
+    - name: Set up Maven
+      uses: stCarolas/[email protected]
+      with:
+        maven-version: 3.8.2
+    - name: Install latest Java shared dependencies
+      shell: bash
+      run: |
+        cd ${{ github.action_path }}/..
+        echo "Install Java shared dependencies"
+        mvn clean install -V --batch-mode --no-transfer-progress -DskipTests
+    - name: Install check
+      shell: bash
+      run: |
+        cd ${{ github.action_path }}
+        echo "Install Unmanaged Dependency Check in $(pwd)"
+        mvn clean install -V --batch-mode --no-transfer-progress -DskipTests
+    - name: Run unmanaged dependency check
+      shell: bash
+      run: |
+        bom_absolute_path=$(realpath "${{ inputs.bom-path }}")
+        cd ${{ github.action_path }}
+        echo "Running Unmanaged Dependency Check against ${bom_absolute_path}"
+        unmanaged_dependencies=$(mvn exec:java -Dexec.args="../pom.xml ${bom_absolute_path}" -q)
+        if [[ "${unmanaged_dependencies}" != "[]" ]]; then
+          echo "This pull request seems to add new third-party dependency, ${unmanaged_dependencies}, among the artifacts listed in ${{ inputs.bom-path }}."
+          echo "Please see go/cloud-sdk-java-dependency-governance."
+          exit 1
+        fi
+
diff --git a/java-shared-dependencies/unmanaged-dependency-check/pom.xml b/java-shared-dependencies/unmanaged-dependency-check/pom.xml
@@ -0,0 +1,85 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+    xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+  <groupId>com.google.cloud</groupId>
+  <artifactId>unmanaged-dependency-check</artifactId>
+  <version>0.0.1-SNAPSHOT</version>
+  <name>Unmanaged dependency check</name>
+  <description>
+
+  </description>
+
+  <properties>
+    <maven.compiler.source>1.8</maven.compiler.source>
+    <maven.compiler.target>1.8</maven.compiler.target>
+    <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+  </properties>
+
+  <build>
+    <plugins>
+      <plugin>
+        <groupId>org.codehaus.mojo</groupId>
+        <artifactId>exec-maven-plugin</artifactId>
+        <version>3.1.1</version>
+        <executions>
+          <execution>
+            <goals>
+              <goal>java</goal>
+            </goals>
+          </execution>
+          <execution>
+            <!-- run the shell script to install test poms so the tests can be executed -->
+            <id>install-test-poms</id>
+            <phase>test-compile</phase>
+            <goals>
+              <goal>exec</goal>
+            </goals>
+            <configuration>
+              <executable>bash</executable>
+              <arguments>
+                <argument>local-install.sh</argument>
+              </arguments>
+              <workingDirectory>src/test/resources</workingDirectory>
+            </configuration>
+          </execution>
+        </executions>
+        <configuration>
+          <mainClass>com.google.cloud.UnmanagedDependencyCheck</mainClass>
+        </configuration>
+      </plugin>
+    </plugins>
+  </build>
+
+  <dependencies>
+    <dependency>
+      <groupId>com.google.cloud.tools</groupId>
+      <artifactId>dependencies</artifactId>
+      <version>1.5.13</version>
+    </dependency>
+    <dependency>
+      <groupId>junit</groupId>
+      <artifactId>junit</artifactId>
+      <version>4.13.2</version>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>com.google.truth</groupId>
+      <artifactId>truth</artifactId>
+      <version>1.1.5</version>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.slf4j</groupId>
+      <artifactId>slf4j-api</artifactId>
+      <version>2.0.9</version>
+    </dependency>
+    <dependency>
+      <groupId>org.slf4j</groupId>
+      <artifactId>slf4j-simple</artifactId>
+      <version>2.0.9</version>
+    </dependency>
+  </dependencies>
+
+</project>
diff --git a/...s/unmanaged-dependency-check/src/main/java/com/google/cloud/UnmanagedDependencyCheck.java b/...s/unmanaged-dependency-check/src/main/java/com/google/cloud/UnmanagedDependencyCheck.java
@@ -0,0 +1,82 @@
+package com.google.cloud;
+
+import static com.google.common.base.Preconditions.checkArgument;
+
+import com.google.cloud.tools.opensource.classpath.ClassPathBuilder;
+import com.google.cloud.tools.opensource.classpath.DependencyMediation;
+import com.google.cloud.tools.opensource.dependencies.Bom;
+import com.google.cloud.tools.opensource.dependencies.MavenRepositoryException;
+import java.nio.file.Paths;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
+import org.eclipse.aether.artifact.Artifact;
+import org.eclipse.aether.version.InvalidVersionSpecificationException;
+
+/**
+ * A utility class to check unmanaged dependencies in BOM.
+ */
+public class UnmanagedDependencyCheck {
+  // regex of handwritten artifacts
+  private final static String downstreamArtifact = "(com.google.cloud:google-cloud-.*)|(com.google.api.grpc:(grpc|proto)-google-cloud-.*)";
+
+
+  /**
+   * @param args An array with two elements.<p> The first string is the path of Java shared
+   * dependencies BOM. <p> The second string is the path of a pom.xml contains BOM.
+   */
+  public static void main(String[] args)
+      throws MavenRepositoryException, InvalidVersionSpecificationException {
+    checkArgument(args.length == 2, "The length of the inputs should be 2");
+    System.out.println(getUnmanagedDependencies(args[0], args[1]));
+  }
+
+  /**
+   * Returns dependency coordinates that are not managed by shared dependency BOM.
+   *
+   * @param sharedDependenciesBomPath the path of shared dependency BOM
+   * @param projectBomPath the path of current project BOM
+   * @return a list of unmanaged dependencies by the given version of shared dependency BOM
+   * @throws MavenRepositoryException thrown if the artifacts in Bom can't be reached in remote or
+   * local Maven repository
+   * @throws InvalidVersionSpecificationException thrown if the shared dependency version can't be
+   * parsed
+   */
+  public static List<String> getUnmanagedDependencies(
+      String sharedDependenciesBomPath, String projectBomPath)
+      throws MavenRepositoryException, InvalidVersionSpecificationException {
+    Set<String> sharedDependencies = getManagedDependencies(sharedDependenciesBomPath);
+    Set<String> managedDependencies = getManagedDependencies(projectBomPath);
+
+    return managedDependencies.stream()
+        .filter(dependency -> !sharedDependencies.contains(dependency))
+        // handwritten artifacts, e.g., com.google.cloud:google-cloud-bigtable, should be excluded.
+        .filter(dependency -> !dependency.matches(downstreamArtifact))
+        .collect(Collectors.toList());
+  }
+
+  private static Set<String> getManagedDependencies(String projectBomPath)
+      throws MavenRepositoryException, InvalidVersionSpecificationException {
+    return getManagedDependenciesFromBom(Bom.readBom(Paths.get(projectBomPath)));
+  }
+
+  private static Set<String> getManagedDependenciesFromBom(Bom bom)
+      throws InvalidVersionSpecificationException {
+    Set<String> res = new HashSet<>();
+    new ClassPathBuilder()
+        .resolve(bom.getManagedDependencies(), true, DependencyMediation.MAVEN)
+        .getClassPath()
+        .forEach(
+            classPath -> {
+              Artifact artifact = classPath.getArtifact();
+              res.add(String.format("%s:%s", artifact.getGroupId(), artifact.getArtifactId()));
+            });
+
+    return res;
+  }
+
+  private UnmanagedDependencyCheck() {
+    throw new IllegalStateException("Utility class");
+  }
+}
diff --git a/...managed-dependency-check/src/test/java/com/google/cloud/UnmanagedDependencyCheckTest.java b/...managed-dependency-check/src/test/java/com/google/cloud/UnmanagedDependencyCheckTest.java
@@ -0,0 +1,43 @@
+package com.google.cloud;
+
+import static com.google.common.truth.Truth.assertThat;
+import static org.junit.Assert.assertTrue;
+
+import com.google.cloud.tools.opensource.dependencies.MavenRepositoryException;
+import com.google.common.collect.ImmutableList;
+import java.util.List;
+import org.eclipse.aether.version.InvalidVersionSpecificationException;
+import org.junit.Test;
+
+public class UnmanagedDependencyCheckTest {
+  @Test
+  public void getUnmanagedDependencyFromSamePomTest()
+      throws MavenRepositoryException, InvalidVersionSpecificationException {
+    String sharedDependenciesBom = "src/test/resources/shared-dependency-3.18.0-pom.xml";
+    List<String> unManagedDependencies =
+        UnmanagedDependencyCheck.getUnmanagedDependencies(sharedDependenciesBom, sharedDependenciesBom);
+    assertTrue(unManagedDependencies.isEmpty());
+  }
+
+  @Test
+  public void getUnmanagedDependencyFromHWBomTest()
+      throws MavenRepositoryException, InvalidVersionSpecificationException {
+    List<String> unManagedDependencies =
+        UnmanagedDependencyCheck.getUnmanagedDependencies(
+            "src/test/resources/shared-dependency-3.18.0-pom.xml", "src/test/resources/bigtable-pom.xml");
+    assertTrue(unManagedDependencies.isEmpty());
+  }
+
+  @Test
+  public void getUnmanagedDependencyFromNestedPomTest()
+      throws MavenRepositoryException, InvalidVersionSpecificationException {
+    List<String> unManagedDependencies =
+        UnmanagedDependencyCheck.getUnmanagedDependencies(
+            "src/test/resources/shared-dependency-3.18.0-pom.xml", "src/test/resources/transitive-dependency-pom.xml");
+    assertThat(unManagedDependencies)
+        .containsAtLeastElementsIn(ImmutableList.of("com.h2database:h2"));
+    // test dependency should be ignored.
+    assertThat(unManagedDependencies)
+        .doesNotContain(ImmutableList.of("com.mysql:mysql-connector-j"));
+  }
+}