Merge branch 'develop' into develop

.github/workflows/docs.yml

@@ -0,0 +1,36 @@
+name: Documentation
+
+on:
+  push:
+    branches:
+      - main
+    tags: "v**"
+    paths:
+      - 'docs/**'
+      - '.github/workflows/docs.yml'
+  workflow_dispatch:
+
+jobs:
+  docs:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/[email protected]
+        with:
+          token: ${{ secrets.SDQ_DEV_DEPLOY_TOKEN }}
+      - uses: actions/[email protected]
+        with:
+          repository: ${{ github.repository }}.wiki
+          path: wiki
+          token: ${{ secrets.SDQ_DEV_DEPLOY_TOKEN }}
+
+      - name: Remove contents in Wiki
+        working-directory: wiki
+        run: ls -A1 | grep -v '.git' | xargs rm -r
+
+      - name: Copy Wiki from Docs folder
+        run: cp -r ./docs/. ./wiki
+
+      - name: Deploy 🚀
+        uses: stefanzweifel/git-auto-commit-action@v4
+        with:
+          repository: wiki

.github/workflows/report-viewer-dev.yml

@@ -34,7 +34,7 @@ jobs:
         working-directory: report-viewer
         run: |
           npm install
-          npm run build
+          npm run build-dev
 
       - name: Deploy 🚀
         uses: JamesIves/[email protected]

.gitignore

@@ -5,7 +5,8 @@ target/
 
 *.class
 
-
+# GitHub
+wiki
 
 # Mobile Tools for Java (J2ME)
 .mtj.tmp/

README.md

@@ -113,9 +113,8 @@ Set<File> submissionDirectories = Set.of(new File("/path/to/rootDir"));
 File baseCode = new File("/path/to/baseCode");
 JPlagOptions options = new JPlagOptions(language, submissionDirectories, Set.of()).withBaseCodeSubmissionDirectory(baseCode);
 
-JPlag jplag = new JPlag(options);
 try {
-    JPlagResult result = jplag.run();
+    JPlagResult result = JPlag.run(options);
 
     // Optional
     ReportObjectFactory reportObjectFactory = new ReportObjectFactory();

cli/pom.xml

@@ -82,6 +82,11 @@
             <artifactId>scheme</artifactId>
             <version>${revision}</version>
         </dependency>
+        <dependency>
+            <groupId>de.jplag</groupId>
+            <artifactId>scxml</artifactId>
+            <version>${revision}</version>
+        </dependency>
         <dependency>
             <groupId>de.jplag</groupId>
             <artifactId>swift</artifactId>

cli/src/main/java/de/jplag/cli/CLI.java

@@ -86,9 +86,7 @@ public static void main(String[] args) {
             CLI cli = new CLI();
             Namespace arguments = cli.parseArguments(args);
             JPlagOptions options = cli.buildOptionsFromArguments(arguments);
-            JPlag jplag = new JPlag(options);
-            logger.debug("JPlag initialized");
-            JPlagResult result = jplag.run();
+            JPlagResult result = JPlag.run(options);
             ReportObjectFactory reportObjectFactory = new ReportObjectFactory();
             reportObjectFactory.createAndSaveReport(result, arguments.getString(RESULT_FOLDER.flagWithoutDash()));
         } catch (ExitException exception) {

cli/src/test/java/de/jplag/cli/LanguageTest.java

@@ -28,7 +28,7 @@ void testInvalidLanguage() throws Exception {
     @Test
     void testLoading() {
         var languages = LanguageLoader.getAllAvailableLanguages();
-        assertEquals(14, languages.size(), "Loaded Languages: " + languages.keySet());
+        assertEquals(15, languages.size(), "Loaded Languages: " + languages.keySet());
     }
 
     @Test
@@ -49,4 +49,4 @@ void testCustomSuffixes() {
         assertEquals(suffixes, options.fileSuffixes());
     }
 
-}
+}

core/pom.xml

@@ -11,6 +11,11 @@
     <artifactId>jplag</artifactId>
 
     <dependencies>
+        <dependency>
+            <groupId>org.jgrapht</groupId>
+            <artifactId>jgrapht-core</artifactId>
+            <version>1.5.2</version>
+        </dependency>
         <dependency>
             <groupId>com.fasterxml.jackson.core</groupId>
             <artifactId>jackson-databind</artifactId>

core/src/main/java/de/jplag/JPlag.java

@@ -30,42 +30,48 @@ private static Version loadVersion() {
 
     private final JPlagOptions options;
 
-    private final Language language;
-    private final ComparisonStrategy comparisonStrategy;
-
     /**
      * Creates and initializes a JPlag instance, parameterized by a set of options.
+     * @deprecated in favor of static {@link #run(JPlagOptions)}.
      * @param options determines the parameterization.
      */
+    @Deprecated(since = "4.3.0")
     public JPlag(JPlagOptions options) {
         this.options = options;
-        language = this.options.language();
-        GreedyStringTiling coreAlgorithm = new GreedyStringTiling(options);
-        comparisonStrategy = new ParallelComparisonStrategy(options, coreAlgorithm);
     }
 
     /**
      * Main procedure, executes the comparison of source code submissions.
+     * @deprecated in favor of static {@link #run(JPlagOptions)}.
      * @return the results of the comparison, specifically the submissions whose similarity exceeds a set threshold.
-     * @throws ExitException if the JPlag exits preemptively.
+     * @throws ExitException if JPlag exits preemptively.
      */
+    @Deprecated(since = "4.3.0")
     public JPlagResult run() throws ExitException {
+        return run(options);
+    }
+
+    /**
+     * Main procedure, executes the comparison of source code submissions.
+     * @param options determines the parameterization.
+     * @return the results of the comparison, specifically the submissions whose similarity exceeds a set threshold.
+     * @throws ExitException if JPlag exits preemptively.
+     */
+    public static JPlagResult run(JPlagOptions options) throws ExitException {
+        GreedyStringTiling coreAlgorithm = new GreedyStringTiling(options);
+        ComparisonStrategy comparisonStrategy = new ParallelComparisonStrategy(options, coreAlgorithm);
         // Parse and validate submissions.
-        SubmissionSetBuilder builder = new SubmissionSetBuilder(language, options);
+        SubmissionSetBuilder builder = new SubmissionSetBuilder(options);
         SubmissionSet submissionSet = builder.buildSubmissionSet();
-
         int submissionCount = submissionSet.numberOfSubmissions();
-        if (submissionCount < 2) {
+        if (submissionCount < 2)
             throw new SubmissionException("Not enough valid submissions! (found " + submissionCount + " valid submissions)");
-        }
 
         // Compare valid submissions.
         JPlagResult result = comparisonStrategy.compareSubmissions(submissionSet);
         if (logger.isInfoEnabled())
             logger.info("Total time for comparing submissions: {}", TimeUtil.formatDuration(result.getDuration()));
-
         result.setClusteringResult(ClusteringFactory.getClusterings(result.getAllComparisons(), options.clusteringOptions()));
-
         return result;
     }
 }

core/src/main/java/de/jplag/Submission.java

@@ -4,15 +4,19 @@
 import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Objects;
+import java.util.Set;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import de.jplag.normalization.TokenStringNormalizer;
+
 /**
  * Represents a single submission. A submission can contain multiple files.
  */
@@ -265,4 +269,33 @@ private static File createErrorDirectory(String... subdirectoryNames) {
         }
         return true;
     }
+
+    /**
+     * Perform token string normalization, which makes the token string invariant to dead code insertion and independent
+     * statement reordering.
+     */
+    void normalize() {
+        List<Integer> originalOrder = getOrder(tokenList);
+        TokenStringNormalizer.normalize(tokenList);
+        List<Integer> normalizedOrder = getOrder(tokenList);
+
+        logger.debug("original line order: {}", originalOrder);
+        logger.debug("line order after normalization: {}", normalizedOrder);
+        Set<Integer> normalizedSet = new HashSet<>(normalizedOrder);
+        List<Integer> removed = originalOrder.stream().filter(l -> !normalizedSet.contains(l)).toList();
+        logger.debug("removed {} line(s): {}", removed.size(), removed);
+    }
+
+    private List<Integer> getOrder(List<Token> tokenList) {
+        List<Integer> order = new ArrayList<>(tokenList.size());  // a little too big
+        int currentLineNumber = tokenList.get(0).getLine();
+        order.add(currentLineNumber);
+        for (Token token : tokenList) {
+            if (token.getLine() != currentLineNumber) {
+                currentLineNumber = token.getLine();
+                order.add(currentLineNumber);
+            }
+        }
+        return order;
+    }
 }

core/src/main/java/de/jplag/SubmissionSet.java

@@ -90,6 +90,10 @@ public List<Submission> getInvalidSubmissions() {
         return invalidSubmissions;
     }
 
+    public void normalizeSubmissions() {
+        submissions.forEach(Submission::normalize);
+    }
+
     private List<Submission> filterValidSubmissions() {
         return allSubmissions.stream().filter(submission -> !submission.hasErrors()).collect(Collectors.toCollection(ArrayList::new));
     }
@@ -118,7 +122,8 @@ private void parseBaseCodeSubmission(Submission baseCode) throws BasecodeExcepti
         if (!baseCode.parse(options.debugParser())) {
             throw new BasecodeException("Could not successfully parse basecode submission!");
         } else if (baseCode.getNumberOfTokens() < options.minimumTokenMatch()) {
-            throw new BasecodeException("Basecode submission contains fewer tokens than minimum match length allows!");
+            throw new BasecodeException(String.format("Basecode submission contains %d token(s), which is less than the minimum match length (%d)!",
+                    baseCode.getNumberOfTokens(), options.minimumTokenMatch()));
         }
         logger.trace("Basecode submission parsed!");
         long duration = System.currentTimeMillis() - startTime;
@@ -149,7 +154,8 @@ private void parseSubmissions(List<Submission> submissions) {
             }
 
             if (submission.getTokenList() != null && submission.getNumberOfTokens() < options.minimumTokenMatch()) {
-                logger.error("Submission {} contains fewer tokens than minimum match length allows!", currentSubmissionName);
+                logger.error("Submission {} contains {} token(s), which is less than the minimum match length ({})!", currentSubmissionName,
+                        submission.getNumberOfTokens(), options.minimumTokenMatch());
                 submission.setTokenList(null);
                 tooShort++;
                 ok = false;

core/src/main/java/de/jplag/SubmissionSetBuilder.java

@@ -31,19 +31,25 @@ public class SubmissionSetBuilder {
 
     private static final Logger logger = LoggerFactory.getLogger(SubmissionSetBuilder.class);
 
-    private final Language language;
     private final JPlagOptions options;
-    private final Set<String> excludedFileNames; // Set of file names to be excluded in comparison.
 
     /**
      * Creates a builder for submission sets.
+     * @deprecated in favor of {@link #SubmissionSetBuilder(JPlagOptions)}.
      * @param language is the language of the submissions.
      * @param options are the configured options.
      */
+    @Deprecated(since = "4.3.0")
     public SubmissionSetBuilder(Language language, JPlagOptions options) {
-        this.language = language;
+        this(options.withLanguageOption(language));
+    }
+
+    /**
+     * Creates a builder for submission sets.
+     * @param options are the configured options.
+     */
+    public SubmissionSetBuilder(JPlagOptions options) {
         this.options = options;
-        this.excludedFileNames = options.excludedFiles();
     }
 
     /**
@@ -209,7 +215,7 @@ private Submission processSubmission(String submissionName, File submissionFile,
         }
 
         submissionFile = makeCanonical(submissionFile, it -> new SubmissionException("Cannot create submission: " + submissionName, it));
-        return new Submission(submissionName, submissionFile, isNew, parseFilesRecursively(submissionFile), language);
+        return new Submission(submissionName, submissionFile, isNew, parseFilesRecursively(submissionFile), options.language());
     }
 
     /**
@@ -254,7 +260,7 @@ private boolean hasValidSuffix(File file) {
      * Checks if a file is excluded or not.
      */
     private boolean isFileExcluded(File file) {
-        return excludedFileNames.stream().anyMatch(excludedName -> file.getName().endsWith(excludedName));
+        return options.excludedFiles().stream().anyMatch(excludedName -> file.getName().endsWith(excludedName));
     }
 
     /**

core/src/main/java/de/jplag/normalization/Edge.java

@@ -0,0 +1,6 @@
+package de.jplag.normalization;
+
+import de.jplag.semantics.Variable;
+
+record Edge(EdgeType type, Variable cause) {
+}

core/src/main/java/de/jplag/normalization/EdgeType.java

@@ -0,0 +1,29 @@
+package de.jplag.normalization;
+
+/**
+ * Enum for types of edges in normalization graph. Given two statements S and T, S comes before T, there is such an edge
+ * between S and T if...
+ */
+enum EdgeType {
+    /**
+     * S writes a variable T reads.
+     */
+    VARIABLE_FLOW,
+    /**
+     * S reads a variable T writes, and S and T are in the same bidirectional block.
+     */
+    VARIABLE_REVERSE_FLOW,
+    /**
+     * S and T access the same variable, and at least one of the two accesses is not a read.
+     */
+    VARIABLE_ORDER,
+    /**
+     * S or T have full position significance, and there is no statement C with full position significance between them.
+     */
+    POSITION_SIGNIFICANCE_FULL,
+    /**
+     * S and T have partial position significance, and there is no statement C with partial position significance between
+     * them.
+     */
+    POSITION_SIGNIFICANCE_PARTIAL
+}

core/src/main/java/de/jplag/normalization/MultipleEdge.java

@@ -0,0 +1,36 @@
+package de.jplag.normalization;
+
+import java.util.HashSet;
+import java.util.Set;
+
+import de.jplag.semantics.Variable;
+
+/**
+ * Models a multiple edge in the normalization graph. Contains multiple edges.
+ */
+class MultipleEdge {
+    private Set<Edge> edges;
+    private boolean isVariableFlow;
+    private boolean isVariableReverseFlow;
+
+    MultipleEdge() {
+        edges = new HashSet<>();
+        isVariableFlow = false;
+    }
+
+    boolean isVariableFlow() {
+        return isVariableFlow;
+    }
+
+    boolean isVariableReverseFlow() {
+        return isVariableReverseFlow;
+    }
+
+    void addEdge(EdgeType type, Variable cause) {
+        if (type == EdgeType.VARIABLE_FLOW)
+            isVariableFlow = true;
+        if (type == EdgeType.VARIABLE_REVERSE_FLOW)
+            isVariableReverseFlow = true;
+        edges.add(new Edge(type, cause));
+    }
+}