Introduce Docgen module to extract website data from source code

By adding a compilation `TaskListener` that extracts data from the Bug pattern
and Refaster rule collection (test) classes and writing to JSON output files
in the target directory. This extraction happens as part of the Maven build
using the `docgen` profile.

docgen/pom.xml

@@ -0,0 +1,53 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <parent>
+        <groupId>tech.picnic.error-prone-support</groupId>
+        <artifactId>error-prone-support</artifactId>
+        <version>0.3.1-SNAPSHOT</version>
+    </parent>
+
+    <artifactId>docgen</artifactId>
+
+    <name>Picnic :: Error Prone Support :: Docgen</name>
+    <description>Docgen.</description>
+
+    <dependencies>
+        <dependency>
+            <groupId>${groupId.error-prone}</groupId>
+            <artifactId>error_prone_annotations</artifactId>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>${groupId.error-prone}</groupId>
+            <artifactId>error_prone_check_api</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>${groupId.error-prone}</groupId>
+            <artifactId>error_prone_core</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>com.fasterxml.jackson.core</groupId>
+            <artifactId>jackson-databind</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>com.google.auto.service</groupId>
+            <artifactId>auto-service-annotations</artifactId>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>com.google.auto.value</groupId>
+            <artifactId>auto-value-annotations</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>com.google.code.findbugs</groupId>
+            <artifactId>jsr305</artifactId>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>com.google.guava</groupId>
+            <artifactId>guava</artifactId>
+        </dependency>
+    </dependencies>
+</project>

docgen/src/main/java/tech/picnic/errorprone/plugin/BugPatternExtractor.java

@@ -0,0 +1,25 @@
+package tech.picnic.errorprone.plugin;
+
+import com.google.errorprone.BugPattern;
+import com.google.errorprone.VisitorState;
+import com.sun.source.tree.ClassTree;
+import com.sun.source.util.TaskEvent;
+import java.util.Arrays;
+import tech.picnic.errorprone.plugin.models.BugPatternData;
+
+public final class BugPatternExtractor implements DocExtractor<BugPatternData> {
+  @Override
+  public BugPatternData extractData(ClassTree tree, TaskEvent taskEvent, VisitorState state) {
+    BugPattern annotation = taskEvent.getTypeElement().getAnnotation(BugPattern.class);
+    return BugPatternData.create(
+        taskEvent.getTypeElement().getSimpleName().toString(),
+        Arrays.toString(annotation.altNames()),
+        annotation.linkType(),
+        annotation.link(),
+        Arrays.toString(annotation.tags()),
+        annotation.summary(),
+        annotation.explanation(),
+        annotation.severity(),
+        annotation.disableable());
+  }
+}

docgen/src/main/java/tech/picnic/errorprone/plugin/BugPatternTestsExtractor.java

@@ -0,0 +1,103 @@
+package tech.picnic.errorprone.plugin;
+
+import static com.google.errorprone.matchers.Matchers.allOf;
+import static com.google.errorprone.matchers.Matchers.hasAnnotation;
+import static com.google.errorprone.matchers.Matchers.instanceMethod;
+
+import com.google.errorprone.VisitorState;
+import com.google.errorprone.annotations.Var;
+import com.google.errorprone.matchers.Matcher;
+import com.google.errorprone.util.ASTHelpers;
+import com.sun.source.tree.ClassTree;
+import com.sun.source.tree.ExpressionTree;
+import com.sun.source.tree.MethodInvocationTree;
+import com.sun.source.tree.MethodTree;
+import com.sun.source.util.TaskEvent;
+import com.sun.source.util.TreeScanner;
+import java.util.ArrayList;
+import java.util.List;
+import tech.picnic.errorprone.plugin.models.BugPatternReplacementTestData;
+import tech.picnic.errorprone.plugin.models.BugPatternTestData;
+
+/** XXX: Write this. */
+// XXX: Take into account `expectUnchanged()`.
+public final class BugPatternTestsExtractor implements DocExtractor<BugPatternTestData> {
+  private static final Matcher<MethodTree> JUNIT_TEST_METHOD =
+      allOf(hasAnnotation("org.junit.jupiter.api.Test"));
+  private static final Matcher<ExpressionTree> IDENTIFICATION_SOURCE_LINES =
+      instanceMethod()
+          .onDescendantOf("com.google.errorprone.CompilationTestHelper")
+          .named("addSourceLines");
+  private static final Matcher<ExpressionTree> REPLACEMENT_INPUT =
+      instanceMethod()
+          .onDescendantOf("com.google.errorprone.BugCheckerRefactoringTestHelper")
+          .named("addInputLines");
+  private static final Matcher<ExpressionTree> REPLACEMENT_OUTPUT =
+      instanceMethod()
+          .onDescendantOf("com.google.errorprone.BugCheckerRefactoringTestHelper.ExpectOutput")
+          .named("addOutputLines");
+
+  @Override
+  public BugPatternTestData extractData(ClassTree tree, TaskEvent taskEvent, VisitorState state) {
+    String name = tree.getSimpleName().toString().replace("Test", "");
+    ScanBugCheckerTestData scanner = new ScanBugCheckerTestData(state);
+
+    tree.getMembers().stream()
+        .filter(MethodTree.class::isInstance)
+        .map(MethodTree.class::cast)
+        .filter(m -> JUNIT_TEST_METHOD.matches(m, state))
+        .forEach(m -> scanner.scan(m, null));
+
+    return BugPatternTestData.create(
+        name, scanner.getIdentificationTests(), scanner.getReplacementTests());
+  }
+
+  private static final class ScanBugCheckerTestData extends TreeScanner<Void, Void> {
+    private final VisitorState state;
+    private final List<String> identificationTests = new ArrayList<>();
+    private final List<BugPatternReplacementTestData> replacementTests = new ArrayList<>();
+
+    // XXX: Using this output field is a bit hacky. Come up with a better solution.
+    @Var private String output;
+
+    ScanBugCheckerTestData(VisitorState state) {
+      this.state = state;
+    }
+
+    public List<String> getIdentificationTests() {
+      return identificationTests;
+    }
+
+    public List<BugPatternReplacementTestData> getReplacementTests() {
+      return replacementTests;
+    }
+
+    @Override
+    public Void visitMethodInvocation(MethodInvocationTree node, Void unused) {
+      if (IDENTIFICATION_SOURCE_LINES.matches(node, state)) {
+        identificationTests.add(getSourceLines(node));
+      } else if (REPLACEMENT_INPUT.matches(node, state)) {
+        replacementTests.add(BugPatternReplacementTestData.create(getSourceLines(node), output));
+      } else if (REPLACEMENT_OUTPUT.matches(node, state)) {
+        output = getSourceLines(node);
+      }
+      return super.visitMethodInvocation(node, unused);
+    }
+
+    private String getSourceLines(MethodInvocationTree tree) {
+      List<? extends ExpressionTree> sourceLines =
+          tree.getArguments().subList(1, tree.getArguments().size());
+      StringBuilder source = new StringBuilder();
+
+      for (ExpressionTree sourceLine : sourceLines) {
+        Object value = ASTHelpers.constValue(sourceLine);
+        if (value == null) {
+          return "";
+        }
+        source.append(value).append('\n');
+      }
+
+      return source.toString();
+    }
+  }
+}

docgen/src/main/java/tech/picnic/errorprone/plugin/DocExtractor.java

@@ -0,0 +1,9 @@
+package tech.picnic.errorprone.plugin;
+
+import com.google.errorprone.VisitorState;
+import com.sun.source.tree.ClassTree;
+import com.sun.source.util.TaskEvent;
+
+public interface DocExtractor<T> {
+  T extractData(ClassTree tree, TaskEvent taskEvent, VisitorState state);
+}

docgen/src/main/java/tech/picnic/errorprone/plugin/DocType.java

@@ -0,0 +1,25 @@
+package tech.picnic.errorprone.plugin;
+
+public enum DocType {
+  BUG_PATTERN("bug-pattern", new BugPatternExtractor()),
+  BUG_PATTERN_TEST("bug-pattern-test", new BugPatternTestsExtractor()),
+  //  REFASTER("refaster", new RefasterExtractor()),
+  REFASTER_TEMPLATE_TEST_INPUT("refaster-test-input", new RefasterTestExtractor()),
+  REFASTER_TEMPLATE_TEST_OUTPUT("refaster-test-output", new RefasterTestExtractor());
+
+  private final String outputFileNamePrefix;
+  private final DocExtractor<?> docExtractor;
+
+  DocType(String outputFileNamePrefix, DocExtractor<?> docExtractor) {
+    this.outputFileNamePrefix = outputFileNamePrefix;
+    this.docExtractor = docExtractor;
+  }
+
+  public String getOutputFileNamePrefix() {
+    return outputFileNamePrefix;
+  }
+
+  public DocExtractor<?> getDocExtractor() {
+    return docExtractor;
+  }
+}

docgen/src/main/java/tech/picnic/errorprone/plugin/Docgen.java

@@ -0,0 +1,21 @@
+package tech.picnic.errorprone.plugin;
+
+import com.google.auto.service.AutoService;
+import com.sun.source.util.JavacTask;
+import com.sun.source.util.Plugin;
+import com.sun.tools.javac.api.BasicJavacTask;
+
+/** XXX: Write. */
+@AutoService(Plugin.class)
+public final class Docgen implements Plugin {
+  @Override
+  public String getName() {
+    return getClass().getSimpleName();
+  }
+
+  @Override
+  public void init(JavacTask javacTask, String... args) {
+    javacTask.addTaskListener(
+        new DocgenTaskListener(((BasicJavacTask) javacTask).getContext(), args[0]));
+  }
+}

docgen/src/main/java/tech/picnic/errorprone/plugin/DocgenTaskListener.java

@@ -0,0 +1,117 @@
+package tech.picnic.errorprone.plugin;
+
+import static tech.picnic.errorprone.plugin.DocType.BUG_PATTERN;
+import static tech.picnic.errorprone.plugin.DocType.BUG_PATTERN_TEST;
+import static tech.picnic.errorprone.plugin.DocType.REFASTER_TEMPLATE_TEST_INPUT;
+import static tech.picnic.errorprone.plugin.DocType.REFASTER_TEMPLATE_TEST_OUTPUT;
+
+import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
+import com.fasterxml.jackson.annotation.PropertyAccessor;
+import com.fasterxml.jackson.core.JsonGenerator;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.SerializationFeature;
+import com.google.errorprone.BugPattern;
+import com.google.errorprone.VisitorState;
+import com.google.errorprone.util.ASTHelpers;
+import com.sun.source.tree.ClassTree;
+import com.sun.source.tree.VariableTree;
+import com.sun.source.util.TaskEvent;
+import com.sun.source.util.TaskListener;
+import com.sun.tools.javac.api.JavacTrees;
+import com.sun.tools.javac.util.Context;
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.Optional;
+import javax.tools.JavaFileObject;
+
+/** XXX: Write this. */
+final class DocgenTaskListener implements TaskListener {
+  private final Context context;
+
+  private final String basePath;
+
+  private final VisitorState state;
+
+  private final ObjectMapper mapper =
+      new ObjectMapper()
+          .setVisibility(PropertyAccessor.FIELD, Visibility.ANY)
+          .configure(SerializationFeature.FAIL_ON_EMPTY_BEANS, false)
+          .configure(JsonGenerator.Feature.AUTO_CLOSE_TARGET, false);
+
+  DocgenTaskListener(Context context, String path) {
+    this.context = context;
+    this.basePath = path.substring(path.indexOf('=') + 1) + "/docs";
+    this.state = VisitorState.createForUtilityPurposes(context);
+
+    // XXX: Move this somewhere else?
+    try {
+      Files.createDirectories(Paths.get(basePath));
+    } catch (IOException e) {
+      throw new RuntimeException(e);
+    }
+  }
+
+  @Override
+  @SuppressWarnings("SystemOut")
+  public void finished(TaskEvent taskEvent) {
+    ClassTree tree = JavacTrees.instance(context).getTree(taskEvent.getTypeElement());
+    JavaFileObject sourceFile = taskEvent.getSourceFile();
+    if (tree == null || sourceFile == null || taskEvent.getKind() != TaskEvent.Kind.ANALYZE) {
+      return;
+    }
+
+    getDocType(tree, sourceFile)
+        .ifPresent(
+            docType ->
+                writeToFile(
+                    docType.getDocExtractor().extractData(tree, taskEvent, state),
+                    docType.getOutputFileNamePrefix(),
+                    getSimpleClassName(sourceFile.getName())));
+  }
+
+  private static Optional<DocType> getDocType(ClassTree tree, JavaFileObject sourceFile) {
+    if (isBugPattern(tree)) {
+      return Optional.of(BUG_PATTERN);
+    } else if (isBugPatternTest(tree)) {
+      return Optional.of(BUG_PATTERN_TEST);
+    } else if (sourceFile.getName().contains("TestInput")) {
+      return Optional.of(REFASTER_TEMPLATE_TEST_INPUT);
+    } else if (sourceFile.getName().contains("TestOutput")) {
+      return Optional.of(REFASTER_TEMPLATE_TEST_OUTPUT);
+    }
+    return Optional.empty();
+  }
+
+  private <T> void writeToFile(T data, String fileName, String name) {
+    File file = new File(basePath + "/" + fileName + "-" + name + ".json");
+    // XXX: Use Path instead of File.
+
+    try (FileWriter fileWriter = new FileWriter(file, true)) {
+      mapper.writeValue(fileWriter, data);
+      fileWriter.write("\n");
+    } catch (IOException e) {
+      e.printStackTrace();
+    }
+  }
+
+  private static boolean isBugPattern(ClassTree tree) {
+    return ASTHelpers.hasDirectAnnotationWithSimpleName(tree, BugPattern.class.getSimpleName());
+  }
+
+  private static boolean isBugPatternTest(ClassTree tree) {
+    return tree.getSimpleName().toString().endsWith("Test")
+        && tree.getMembers().stream()
+            .filter(VariableTree.class::isInstance)
+            .map(VariableTree.class::cast)
+            .anyMatch(vt -> vt.getType().toString().equals("BugCheckerRefactoringTestHelper"));
+  }
+
+  private static String getSimpleClassName(String path) {
+    int index = path.lastIndexOf('/');
+    String fileName = path.substring(index + 1);
+    return fileName.replace(".java", "");
+  }
+}

docgen/src/main/java/tech/picnic/errorprone/plugin/RefasterTestExtractor.java

@@ -0,0 +1,35 @@
+package tech.picnic.errorprone.plugin;
+
+import static com.google.common.collect.ImmutableList.toImmutableList;
+
+import com.google.common.collect.ImmutableList;
+import com.google.errorprone.VisitorState;
+import com.sun.source.tree.ClassTree;
+import com.sun.source.tree.MethodTree;
+import com.sun.source.util.TaskEvent;
+import tech.picnic.errorprone.plugin.models.RefasterTemplateCollectionTestData;
+import tech.picnic.errorprone.plugin.models.RefasterTemplateTestData;
+
+public final class RefasterTestExtractor
+    implements DocExtractor<RefasterTemplateCollectionTestData> {
+  @Override
+  public RefasterTemplateCollectionTestData extractData(
+      ClassTree tree, TaskEvent taskEvent, VisitorState state) {
+    String templateCollectionName = tree.getSimpleName().toString().replace("Test", "");
+    boolean isInput = taskEvent.getSourceFile().getName().contains("Input");
+
+    ImmutableList<RefasterTemplateTestData> templateTests =
+        tree.getMembers().stream()
+            .filter(MethodTree.class::isInstance)
+            .map(MethodTree.class::cast)
+            .filter(m -> m.getName().toString().startsWith("test"))
+            .map(
+                m ->
+                    RefasterTemplateTestData.create(
+                        m.getName().toString().replace("test", ""), m.toString()))
+            .collect(toImmutableList());
+
+    return RefasterTemplateCollectionTestData.create(
+        templateCollectionName, isInput, templateTests);
+  }
+}