diff --git a/CHANGELOG.md b/CHANGELOG.md index 4bad6edb909..dea34f011c6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ Note that this project **does not** adhere to [Semantic Versioning](http://semve - We added a fetcher for [Biodiversity Heritage Library](https://www.biodiversitylibrary.org/). [8539](https://github.com/JabRef/jabref/issues/8539) - We added support for multiple messages in the snackbar. [#7340](https://github.com/JabRef/jabref/issues/7340) +- We added an extra option in the 'Find Unlinked Files' dialog view to ignore unnecessary files like Thumbs.db, DS_Store, etc. [koppor#373](https://github.com/koppor/jabref/issues/373) - JabRef now writes log files. Linux: `$home/.cache/jabref/logs/version`, Windows: `%APPDATA%\..\Local\harawata\jabref\version\logs`, Mac: `Users/.../Library/Logs/jabref/version` - We added an importer for Citavi backup files, support ".ctv5bak" and ".ctv6bak" file formats. [#8322](https://github.com/JabRef/jabref/issues/8322) diff --git a/src/main/java/org/jabref/gui/externalfiles/ChainedFilters.java b/src/main/java/org/jabref/gui/externalfiles/ChainedFilters.java new file mode 100644 index 00000000000..4162c9c674f --- /dev/null +++ b/src/main/java/org/jabref/gui/externalfiles/ChainedFilters.java @@ -0,0 +1,35 @@ +package org.jabref.gui.externalfiles; + +import java.io.IOException; +import java.nio.file.DirectoryStream; +import java.nio.file.Path; +import java.util.Arrays; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Chains the given filters - if ALL of them accept, the result is also accepted + */ +public class ChainedFilters implements DirectoryStream.Filter { + + private static final Logger LOGGER = LoggerFactory.getLogger(ChainedFilters.class); + + private DirectoryStream.Filter[] filters; + + public ChainedFilters(DirectoryStream.Filter... filters) { + this.filters = filters; + } + + @Override + public boolean accept(Path entry) throws IOException { + return Arrays.stream(filters).allMatch(filter -> { + try { + return filter.accept(entry); + } catch (IOException e) { + LOGGER.error("Could not apply filter", e); + return true; + } + }); + } +} diff --git a/src/main/java/org/jabref/gui/externalfiles/FileFilterUtils.java b/src/main/java/org/jabref/gui/externalfiles/FileFilterUtils.java index 28e1f20929a..6c9e48d13e3 100644 --- a/src/main/java/org/jabref/gui/externalfiles/FileFilterUtils.java +++ b/src/main/java/org/jabref/gui/externalfiles/FileFilterUtils.java @@ -16,10 +16,10 @@ public class FileFilterUtils { private static final Logger LOGGER = LoggerFactory.getLogger(FileFilterUtils.class); - + /* Returns the last edited time of a file as LocalDateTime. */ public static LocalDateTime getFileTime(Path path) { - FileTime lastEditedTime = null; + FileTime lastEditedTime; try { lastEditedTime = Files.getLastModifiedTime(path); } catch (IOException e) { @@ -33,28 +33,28 @@ public static LocalDateTime getFileTime(Path path) { return localDateTime; } - /* Returns true if a file with a specific path + /* Returns true if a file with a specific path * was edited during the last 24 hours. */ public boolean isDuringLastDay(LocalDateTime fileEditTime) { LocalDateTime NOW = LocalDateTime.now(ZoneId.systemDefault()); return fileEditTime.isAfter(NOW.minusHours(24)); } - /* Returns true if a file with a specific path + /* Returns true if a file with a specific path * was edited during the last 7 days. */ public boolean isDuringLastWeek(LocalDateTime fileEditTime) { LocalDateTime NOW = LocalDateTime.now(ZoneId.systemDefault()); return fileEditTime.isAfter(NOW.minusDays(7)); } - /* Returns true if a file with a specific path + /* Returns true if a file with a specific path * was edited during the last 30 days. */ public boolean isDuringLastMonth(LocalDateTime fileEditTime) { LocalDateTime NOW = LocalDateTime.now(ZoneId.systemDefault()); return fileEditTime.isAfter(NOW.minusDays(30)); } - /* Returns true if a file with a specific path + /* Returns true if a file with a specific path * was edited during the last 365 days. */ public boolean isDuringLastYear(LocalDateTime fileEditTime) { LocalDateTime NOW = LocalDateTime.now(ZoneId.systemDefault()); @@ -75,8 +75,10 @@ public static boolean filterByDate(Path path, DateRange filter) { return isInDateRange; } - /* Sorts a list of Path objects according to the last edited date - * of their corresponding files, from newest to oldest. */ + /** + * Sorts a list of Path objects according to the last edited date + * of their corresponding files, from newest to oldest. + */ public List sortByDateAscending(List files) { return files.stream() .sorted(Comparator.comparingLong(file -> FileFilterUtils.getFileTime(file) @@ -86,8 +88,10 @@ public List sortByDateAscending(List files) { .collect(Collectors.toList()); } - /* Sorts a list of Path objects according to the last edited date - * of their corresponding files, from oldest to newest. */ + /** + * Sorts a list of Path objects according to the last edited date + * of their corresponding files, from oldest to newest. + */ public List sortByDateDescending(List files) { return files.stream() .sorted(Comparator.comparingLong(file -> -FileFilterUtils.getFileTime(file) @@ -97,8 +101,10 @@ public List sortByDateDescending(List files) { .collect(Collectors.toList()); } - /* Sorts a list of Path objects according to the last edited date - * the order depends on the specified sorter type. */ + /** + * Sorts a list of Path objects according to the last edited date + * the order depends on the specified sorter type. + */ public static List sortByDate(List files, ExternalFileSorter sortType) { FileFilterUtils fileFilter = new FileFilterUtils(); List sortedFiles = switch (sortType) { diff --git a/src/main/java/org/jabref/gui/externalfiles/GitIgnoreFileFilter.java b/src/main/java/org/jabref/gui/externalfiles/GitIgnoreFileFilter.java new file mode 100644 index 00000000000..64f01de977b --- /dev/null +++ b/src/main/java/org/jabref/gui/externalfiles/GitIgnoreFileFilter.java @@ -0,0 +1,66 @@ +package org.jabref.gui.externalfiles; + +import java.io.IOException; +import java.nio.file.DirectoryStream; +import java.nio.file.FileSystems; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.PathMatcher; +import java.util.HashSet; +import java.util.Set; +import java.util.stream.Collectors; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static java.util.function.Predicate.not; + +public class GitIgnoreFileFilter implements DirectoryStream.Filter { + + private static final Logger LOGGER = LoggerFactory.getLogger(GitIgnoreFileFilter.class); + + private Set gitIgnorePatterns; + + public GitIgnoreFileFilter(Path path) { + Path currentPath = path; + while ((currentPath != null) && !Files.exists(currentPath.resolve(".gitignore"))) { + currentPath = currentPath.getParent(); + } + if (currentPath == null) { + // we did not find any gitignore, lets use the default + gitIgnorePatterns = Set.of(".git", ".DS_Store", "desktop.ini", "Thumbs.db").stream() + // duplicate code as below + .map(line -> "glob:" + line) + .map(matcherString -> FileSystems.getDefault().getPathMatcher(matcherString)) + .collect(Collectors.toSet()); + } else { + Path gitIgnore = currentPath.resolve(".gitignore"); + try { + Set plainGitIgnorePatternsFromGitIgnoreFile = Files.readAllLines(gitIgnore).stream() + .map(line -> line.trim()) + .filter(not(String::isEmpty)) + .filter(line -> !line.startsWith("#")) + // convert to Java syntax for Glob patterns + .map(line -> "glob:" + line) + .map(matcherString -> FileSystems.getDefault().getPathMatcher(matcherString)) + .collect(Collectors.toSet()); + gitIgnorePatterns = new HashSet<>(plainGitIgnorePatternsFromGitIgnoreFile); + // we want to ignore ".gitignore" itself + gitIgnorePatterns.add(FileSystems.getDefault().getPathMatcher("glob:.gitignore")); + } catch (IOException e) { + LOGGER.info("Could not read .gitignore from {}", gitIgnore, e); + gitIgnorePatterns = Set.of(); + } + } + } + + @Override + public boolean accept(Path path) throws IOException { + // We assume that git does not stop at a patern, but tries all. We implement that behavior + return gitIgnorePatterns.stream().noneMatch(filter -> + // we need this one for "*.png" + filter.matches(path.getFileName()) || + // we need this one for "**/*.png" + filter.matches(path)); + } +} diff --git a/src/main/java/org/jabref/gui/externalfiles/UnlinkedFilesCrawler.java b/src/main/java/org/jabref/gui/externalfiles/UnlinkedFilesCrawler.java index 53ebad10fc5..ef26c177a25 100644 --- a/src/main/java/org/jabref/gui/externalfiles/UnlinkedFilesCrawler.java +++ b/src/main/java/org/jabref/gui/externalfiles/UnlinkedFilesCrawler.java @@ -66,52 +66,76 @@ protected FileNodeViewModel call() throws IOException { * 'state' must be set to 1, to keep the recursion running. When the states value changes, the method will resolve * its recursion and return what it has saved so far. *
- * The files are filtered according to the {@link DateRange} filter value + * The files are filtered according to the {@link DateRange} filter value * and then sorted according to the {@link ExternalFileSorter} value. * + * @param unlinkedPDFFileFilter contains a BibDatabaseContext which is used to determine whether the file is linked + * + * @return FileNodeViewModel containing the data of the current directory and all subdirectories * @throws IOException if directory is not a directory or empty */ - private FileNodeViewModel searchDirectory(Path directory, UnlinkedPDFFileFilter fileFilter) throws IOException { + FileNodeViewModel searchDirectory(Path directory, UnlinkedPDFFileFilter unlinkedPDFFileFilter) throws IOException { // Return null if the directory is not valid. if ((directory == null) || !Files.isDirectory(directory)) { throw new IOException(String.format("Invalid directory for searching: %s", directory)); } - FileNodeViewModel parent = new FileNodeViewModel(directory); - Map> fileListPartition; + FileNodeViewModel fileNodeViewModelForCurrentDirectory = new FileNodeViewModel(directory); - try (Stream filesStream = StreamSupport.stream(Files.newDirectoryStream(directory, fileFilter).spliterator(), false)) { - fileListPartition = filesStream.collect(Collectors.partitioningBy(Files::isDirectory)); + // Map from isDirectory (true/false) to full path + // Result: Contains only files not matching the filter (i.e., PDFs not linked and files not ignored) + // Filters: + // 1. UnlinkedPDFFileFilter + // 2. GitIgnoreFilter + ChainedFilters filters = new ChainedFilters(unlinkedPDFFileFilter, new GitIgnoreFileFilter(directory)); + Map> directoryAndFilePartition; + try (Stream filesStream = StreamSupport.stream(Files.newDirectoryStream(directory, filters).spliterator(), false)) { + directoryAndFilePartition = filesStream.collect(Collectors.partitioningBy(Files::isDirectory)); } catch (IOException e) { - LOGGER.error(String.format("%s while searching files: %s", e.getClass().getName(), e.getMessage())); - return parent; + LOGGER.error("Error while searching files", e); + return fileNodeViewModelForCurrentDirectory; } + List subDirectories = directoryAndFilePartition.get(true); + List files = directoryAndFilePartition.get(false); - List subDirectories = fileListPartition.get(true); - List files = new ArrayList<>(fileListPartition.get(false)); - int fileCount = 0; + // at this point, only unlinked PDFs AND unignored files are contained - for (Path subDirectory : subDirectories) { - FileNodeViewModel subRoot = searchDirectory(subDirectory, fileFilter); + // initially, we find no files at all + int fileCountOfSubdirectories = 0; + // now we crawl into the found subdirectories first (!) + for (Path subDirectory : subDirectories) { + FileNodeViewModel subRoot = searchDirectory(subDirectory, unlinkedPDFFileFilter); if (!subRoot.getChildren().isEmpty()) { - fileCount += subRoot.getFileCount(); - parent.getChildren().add(subRoot); + fileCountOfSubdirectories += subRoot.getFileCount(); + fileNodeViewModelForCurrentDirectory.getChildren().add(subRoot); } } + // now we have the data of all subdirectories + // it is stored in fileNodeViewModelForCurrentDirectory.getChildren() + + // now we handle the files in the current directory + // filter files according to last edited date. - List filteredFiles = new ArrayList(); + // Note that we do not use the "StreamSupport.stream" filtering functionality, because refactoring the code to that would lead to more code + List resultingFiles = new ArrayList<>(); for (Path path : files) { if (FileFilterUtils.filterByDate(path, dateFilter)) { - filteredFiles.add(path); + resultingFiles.add(path); } } + // sort files according to last edited date. - filteredFiles = FileFilterUtils.sortByDate(filteredFiles, sorter); - parent.setFileCount(filteredFiles.size() + fileCount); - parent.getChildren().addAll(filteredFiles.stream() + resultingFiles = FileFilterUtils.sortByDate(resultingFiles, sorter); + + // the count of all files is the count of the found files in current directory plus the count of all files in the subdirectories + fileNodeViewModelForCurrentDirectory.setFileCount(resultingFiles.size() + fileCountOfSubdirectories); + + // create and add FileNodeViewModel to the FileNodeViewModel for the current directory + fileNodeViewModelForCurrentDirectory.getChildren().addAll(resultingFiles.stream() .map(FileNodeViewModel::new) .collect(Collectors.toList())); - return parent; + + return fileNodeViewModelForCurrentDirectory; } } diff --git a/src/main/java/org/jabref/gui/externalfiles/UnlinkedPDFFileFilter.java b/src/main/java/org/jabref/gui/externalfiles/UnlinkedPDFFileFilter.java index 0cdb7263263..0532801d222 100644 --- a/src/main/java/org/jabref/gui/externalfiles/UnlinkedPDFFileFilter.java +++ b/src/main/java/org/jabref/gui/externalfiles/UnlinkedPDFFileFilter.java @@ -32,7 +32,6 @@ public UnlinkedPDFFileFilter(DirectoryStream.Filter fileFilter, BibDatabas @Override public boolean accept(Path pathname) throws IOException { - if (Files.isDirectory(pathname)) { return true; } else { diff --git a/src/main/java/org/jabref/gui/util/FileNodeViewModel.java b/src/main/java/org/jabref/gui/util/FileNodeViewModel.java index 2c7d4e1a103..2410ad4b67c 100644 --- a/src/main/java/org/jabref/gui/util/FileNodeViewModel.java +++ b/src/main/java/org/jabref/gui/util/FileNodeViewModel.java @@ -7,6 +7,7 @@ import java.time.LocalDateTime; import java.time.ZoneId; import java.time.format.DateTimeFormatter; +import java.util.Objects; import javafx.beans.property.ReadOnlyListWrapper; import javafx.collections.FXCollections; @@ -94,4 +95,21 @@ public String toString() { this.children, this.fileCount); } + + @Override + public int hashCode() { + return Objects.hash(children, fileCount, path); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (!(obj instanceof FileNodeViewModel)) { + return false; + } + FileNodeViewModel other = (FileNodeViewModel) obj; + return Objects.equals(children, other.children) && (fileCount == other.fileCount) && Objects.equals(path, other.path); + } } diff --git a/src/test/java/org/jabref/gui/externalfiles/FileFilterUtilsTest.java b/src/test/java/org/jabref/gui/externalfiles/FileFilterUtilsTest.java index a6888ab8d3a..9af4b1364bb 100755 --- a/src/test/java/org/jabref/gui/externalfiles/FileFilterUtilsTest.java +++ b/src/test/java/org/jabref/gui/externalfiles/FileFilterUtilsTest.java @@ -5,7 +5,9 @@ import java.nio.file.attribute.FileTime; import java.time.LocalDateTime; import java.util.ArrayList; +import java.util.HashSet; import java.util.List; +import java.util.Set; import java.util.stream.Collectors; import org.junit.jupiter.api.BeforeEach; @@ -64,20 +66,20 @@ public void isDuringLastYearPositiveTest() { @Nested class SortingTests { - private List files = new ArrayList(); - private List expectedSortByDateAscending = new ArrayList(); - private List expectedSortByDateDescending = new ArrayList(); - private List wrongOrder = new ArrayList(); + private final List files = new ArrayList<>(); + private final List expectedSortByDateAscending = new ArrayList<>(); + private final List expectedSortByDateDescending = new ArrayList<>(); + private final List wrongOrder = new ArrayList<>(); /* Initialize the directory and files used in the sorting tests, and change their last edited dates. */ @BeforeEach public void setUp(@TempDir Path tempDir) throws Exception { - + Path firstPath = tempDir.resolve("firstFile.pdf"); Path secondPath = tempDir.resolve("secondFile.pdf"); Path thirdPath = tempDir.resolve("thirdFile.pdf"); Path fourthPath = tempDir.resolve("fourthFile.pdf"); - + Files.createFile(firstPath); Files.createFile(secondPath); Files.createFile(thirdPath); @@ -88,19 +90,19 @@ public void setUp(@TempDir Path tempDir) throws Exception { Files.setLastModifiedTime(secondPath, FileTime.fromMillis(5)); Files.setLastModifiedTime(thirdPath, FileTime.fromMillis(1)); Files.setLastModifiedTime(fourthPath, FileTime.fromMillis(2)); - + // fill the list to be sorted by the tests. files.add(firstPath); files.add(secondPath); files.add(thirdPath); files.add(fourthPath); - + // fill the expected values lists. expectedSortByDateAscending.add(thirdPath.toString()); expectedSortByDateAscending.add(fourthPath.toString()); expectedSortByDateAscending.add(secondPath.toString()); expectedSortByDateAscending.add(firstPath.toString()); - + expectedSortByDateDescending.add(firstPath.toString()); expectedSortByDateDescending.add(secondPath.toString()); expectedSortByDateDescending.add(fourthPath.toString()); @@ -111,7 +113,7 @@ public void setUp(@TempDir Path tempDir) throws Exception { wrongOrder.add(thirdPath.toString()); wrongOrder.add(fourthPath.toString()); } - + @Test public void sortByDateAscendingPositiveTest() { List sortedPaths = fileFilterUtils @@ -131,7 +133,7 @@ public void sortByDateAscendingNegativeTest() { .collect(Collectors.toList()); assertNotEquals(sortedPaths, wrongOrder); } - + @Test public void sortByDateDescendingPositiveTest() { List sortedPaths = fileFilterUtils @@ -152,4 +154,43 @@ public void testSortByDateDescendingNegativeTest() { assertNotEquals(sortedPaths, wrongOrder); } } + + @Nested + class filteringTests { + private final List files = new ArrayList<>(); + private final List targetFiles = new ArrayList<>(); + private final Set ignoreFileSet = new HashSet<>(); + + @BeforeEach + public void setUp(@TempDir Path tempDir) throws Exception { + ignoreFileSet.add(".DS_Store"); + ignoreFileSet.add("Thumbs.db"); + + Path firstPath = tempDir.resolve("firstFile.pdf"); + Path secondPath = tempDir.resolve("secondFile.pdf"); + Path thirdPath = tempDir.resolve("thirdFile.pdf"); + Path fourthPath = tempDir.resolve("fourthFile.pdf"); + Path fifthPath = tempDir.resolve(".DS_Store"); + Path sixthPath = tempDir.resolve("Thumbs.db"); + + Files.createFile(firstPath); + Files.createFile(secondPath); + Files.createFile(thirdPath); + Files.createFile(fourthPath); + Files.createFile(fifthPath); + Files.createFile(sixthPath); + + files.add(firstPath); + files.add(secondPath); + files.add(thirdPath); + files.add(fourthPath); + files.add(fifthPath); + files.add(sixthPath); + + targetFiles.add(firstPath); + targetFiles.add(secondPath); + targetFiles.add(thirdPath); + targetFiles.add(fourthPath); + } + } } diff --git a/src/test/java/org/jabref/gui/externalfiles/GitIgnoreFileFilterTest.java b/src/test/java/org/jabref/gui/externalfiles/GitIgnoreFileFilterTest.java new file mode 100644 index 00000000000..2504ffb55b7 --- /dev/null +++ b/src/test/java/org/jabref/gui/externalfiles/GitIgnoreFileFilterTest.java @@ -0,0 +1,50 @@ +package org.jabref.gui.externalfiles; + +import java.nio.file.Files; +import java.nio.file.Path; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class GitIgnoreFileFilterTest { + + @Test + public void checkSimpleGitIgnore(@TempDir Path dir) throws Exception { + Files.writeString(dir.resolve(".gitignore"), """ + *.png + """); + GitIgnoreFileFilter gitIgnoreFileFilter = new GitIgnoreFileFilter(dir); + assertFalse(gitIgnoreFileFilter.accept(dir.resolve("test.png"))); + } + + @Test + public void checkSimpleGitIgnoreWithAllowing(@TempDir Path dir) throws Exception { + Files.writeString(dir.resolve(".gitignore"), """ + !*.png + """); + GitIgnoreFileFilter gitIgnoreFileFilter = new GitIgnoreFileFilter(dir); + assertTrue(gitIgnoreFileFilter.accept(dir.resolve("test.png"))); + } + + @Test + public void checkSimpleGitIgnoreWithOverwritingDefs(@TempDir Path dir) throws Exception { + Files.writeString(dir.resolve(".gitignore"), """ + !*.png + *.png + """); + GitIgnoreFileFilter gitIgnoreFileFilter = new GitIgnoreFileFilter(dir); + assertFalse(gitIgnoreFileFilter.accept(dir.resolve("test.png"))); + } + + @Test + public void checkDirectoryGitIgnore(@TempDir Path dir) throws Exception { + Files.writeString(dir.resolve(".gitignore"), """ + **/*.png + """); + GitIgnoreFileFilter gitIgnoreFileFilter = new GitIgnoreFileFilter(dir); + assertFalse(gitIgnoreFileFilter.accept(dir.resolve("test.png"))); + } +} diff --git a/src/test/java/org/jabref/gui/externalfiles/UnlinkedFilesCrawlerTest.java b/src/test/java/org/jabref/gui/externalfiles/UnlinkedFilesCrawlerTest.java new file mode 100644 index 00000000000..3d53ae3ef59 --- /dev/null +++ b/src/test/java/org/jabref/gui/externalfiles/UnlinkedFilesCrawlerTest.java @@ -0,0 +1,38 @@ +package org.jabref.gui.externalfiles; + +import java.nio.file.Files; +import java.nio.file.Path; + +import org.jabref.gui.util.FileNodeViewModel; +import org.jabref.model.database.BibDatabaseContext; +import org.jabref.preferences.FilePreferences; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +class UnlinkedFilesCrawlerTest { + + @Test + public void minimalGitIgnore(@TempDir Path testRoot) throws Exception { + Files.writeString(testRoot.resolve(".gitignore"), """ + *.png + """); + Path subDir = testRoot.resolve("subdir"); + Files.createDirectories(subDir); + Files.createFile(subDir.resolve("test.png")); + + UnlinkedPDFFileFilter unlinkedPDFFileFilter = mock(UnlinkedPDFFileFilter.class); + when(unlinkedPDFFileFilter.accept(any(Path.class))).thenReturn(true); + + UnlinkedFilesCrawler unlinkedFilesCrawler = new UnlinkedFilesCrawler(testRoot, unlinkedPDFFileFilter, DateRange.ALL_TIME, ExternalFileSorter.DEFAULT, mock(BibDatabaseContext.class), mock(FilePreferences.class)); + + FileNodeViewModel fileNodeViewModel = unlinkedFilesCrawler.searchDirectory(testRoot, unlinkedPDFFileFilter); + + assertEquals(new FileNodeViewModel(testRoot), fileNodeViewModel); + } +}