Add jabkit get-fulltexts

.jbang/JabKitLauncher.java

@@ -32,6 +32,7 @@
 //SOURCES ../jabkit/src/main/java/org/jabref/toolkit/commands/GenerateCitationKeys.java
 //SOURCES ../jabkit/src/main/java/org/jabref/toolkit/commands/GetCitedWorks.java
 //SOURCES ../jabkit/src/main/java/org/jabref/toolkit/commands/GetCitingWorks.java
+//SOURCES ../jabkit/src/main/java/org/jabref/toolkit/commands/GetFulltexts.java
 //SOURCES ../jabkit/src/main/java/org/jabref/toolkit/commands/Git.java
 //SOURCES ../jabkit/src/main/java/org/jabref/toolkit/commands/GitMergeDriver.java
 //SOURCES ../jabkit/src/main/java/org/jabref/toolkit/commands/InputOption.java

CHANGELOG.md

@@ -12,6 +12,7 @@ Note that this project **does not** adhere to [Semantic Versioning](https://semv
 ### Added
 
 - We added `jabkit git merge-driver`, a Git merge driver that merges `.bib` files semantically. [#16838](https://github.com/JabRef/jabref/pull/16838)
+- We added `jabkit get-fulltexts` to download and link full text PDFs, reporting the result per entry. [#770](https://github.com/JabRef/jabref-koppor/pull/770)
 
 ### Changed
 

docs/requirements/cli.md

@@ -93,4 +93,14 @@ are parsed correctly by the GitHub Actions runner.
 
 Needs: impl
 
+## Full text download reports each entry
+`req~jabkit.cli.get-fulltexts-report~1`
+
+`jabkit get-fulltexts` updates the given library in place (or writes to `--output`).
+It prints one line per entry saying whether its full text document was downloaded, not found,
+skipped, or failed, followed by a summary with the counts.
+Entries that already link a local PDF are skipped without contacting a publisher.
+
+Needs: impl
+
 <!-- markdownlint-disable-file MD022 -->

jabkit/src/main/java/org/jabref/toolkit/commands/GetFulltexts.java

@@ -0,0 +1,115 @@
+package org.jabref.toolkit.commands;
+
+import java.nio.file.Path;
+import java.util.concurrent.Callable;
+
+import org.jabref.logic.externalfiles.FulltextDownloader;
+import org.jabref.logic.importer.ParserResult;
+import org.jabref.logic.l10n.Localization;
+import org.jabref.logic.preferences.CliPreferences;
+import org.jabref.logic.util.StandardFileType;
+import org.jabref.model.database.BibDatabaseContext;
+import org.jabref.model.entry.BibEntry;
+import org.jabref.model.entry.LinkedFile;
+import org.jabref.toolkit.exception.ExportServiceException;
+import org.jabref.toolkit.exception.ImportServiceException;
+import org.jabref.toolkit.service.ExportService;
+import org.jabref.toolkit.service.ImportService;
+
+import picocli.CommandLine;
+import picocli.CommandLine.Command;
+import picocli.CommandLine.Mixin;
+import picocli.CommandLine.Option;
+import picocli.CommandLine.ParentCommand;
+
+// [impl->req~jabkit.cli.get-fulltexts-report~1]
+@Command(name = "get-fulltexts", description = "Download the full text PDFs of the entries of a library and link them.")
+class GetFulltexts implements Callable<Integer> {
+
+    protected FulltextDownloader fulltextDownloader;
+
+    @ParentCommand
+    private JabKit argumentProcessor;
+
+    @Mixin
+    private JabKit.SharedOptions sharedOptions;
+
+    @Mixin
+    private InputOption inputOption = new InputOption();
+
+    @Option(names = "--output", description = "Output .bib file (default: the input file is updated)")
+    private Path outputFile;
+
+    void initFields() {
+        CliPreferences preferences = argumentProcessor.cliPreferences;
+        fulltextDownloader = new FulltextDownloader(preferences.getImportFormatPreferences(), preferences.getImporterPreferences(), preferences.getFilePreferences());
+    }
+
+    @Override
+    public Integer call() throws ImportServiceException, ExportServiceException {
+        initFields();
+        CliPreferences preferences = argumentProcessor.cliPreferences;
+        Path inputFile = inputOption.getInputFile(preferences);
+        ParserResult parserResult = ImportService.importBibTexFile(inputFile, preferences, sharedOptions.porcelain);
+        BibDatabaseContext databaseContext = parserResult.getDatabaseContext();
+        // Relative file directories and the "store files next to the library" preference resolve against it
+        databaseContext.setDatabasePath(inputFile.toAbsolutePath());
+
+        int downloaded = 0;
+        int notFound = 0;
+        int skipped = 0;
+        int failed = 0;
+        for (BibEntry entry : databaseContext.getEntries()) {
+            String citationKey = entry.getCitationKey().orElse(Localization.lang("undefined"));
+            // Keeps a re-run from sending requests to publishers for entries it already handled
+            if (hasLocalPdf(entry)) {
+                System.out.println(Localization.lang("Full text document for entry %0 already linked.", citationKey));
+                skipped++;
+                continue;
+            }
+            switch (fulltextDownloader.download(databaseContext, entry)) {
+                case FulltextDownloader.Result.Downloaded downloadedFile -> {
+                    System.out.println(Localization.lang("Downloaded full text document for entry %0: %1", citationKey, downloadedFile.file()));
+                    downloaded++;
+                }
+                case FulltextDownloader.Result.NotFound _ -> {
+                    System.out.println(Localization.lang("No full text document found for entry %0.", citationKey));
+                    notFound++;
+                }
+                case FulltextDownloader.Result.AlreadyLinked _ -> {
+                    System.out.println(Localization.lang("Full text document for entry %0 already linked.", citationKey));
+                    skipped++;
+                }
+                case FulltextDownloader.Result.Duplicate _ -> {
+                    System.out.println(Localization.lang("Full text document for entry %0 is a duplicate of an existing file.", citationKey));
+                    skipped++;
+                }
+                case FulltextDownloader.Result.Failed failure -> {
+                    System.out.println(Localization.lang("Could not download the full text document for entry %0: %1", citationKey, failure.message()));
+                    failed++;
+                }
+                case FulltextDownloader.Result.NoFileDirectory _ -> {
+                    System.err.println(Localization.lang("No existing file directory to download the full text documents to."));
+                    return CommandLine.ExitCode.SOFTWARE;
+                }
+            }
+        }
+
+        if (!sharedOptions.porcelain) {
+            System.out.println(Localization.lang("Full text documents: %0 downloaded, %1 not found, %2 skipped, %3 failed.",
+                    downloaded, notFound, skipped, failed));
+        }
+
+        if (downloaded > 0 || outputFile != null) {
+            ExportService.create(preferences, sharedOptions.porcelain)
+                         .saveDatabaseContext(databaseContext, outputFile != null ? outputFile : inputFile);
+        }
+        return failed > 0 ? CommandLine.ExitCode.SOFTWARE : CommandLine.ExitCode.OK;
+    }
+
+    private static boolean hasLocalPdf(BibEntry entry) {
+        return entry.getFiles().stream()
+                    .anyMatch(file -> !LinkedFile.isOnlineLink(file.getLink())
+                            && StandardFileType.PDF.getName().equalsIgnoreCase(file.getFileType()));
+    }
+}

jabkit/src/main/java/org/jabref/toolkit/commands/JabKit.java

@@ -23,6 +23,7 @@
                 GenerateBibFromAux.class,
                 GetCitedWorks.class,
                 GetCitingWorks.class,
+                GetFulltexts.class,
                 Git.class,
                 Pdf.class,
                 Preferences.class,

jabkit/src/test/java/org/jabref/toolkit/commands/GetFulltextsTest.java

@@ -0,0 +1,77 @@
+package org.jabref.toolkit.commands;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+
+import org.jabref.logic.externalfiles.FulltextDownloader;
+import org.jabref.model.entry.BibEntry;
+import org.jabref.model.entry.LinkedFile;
+import org.jabref.toolkit.util.CapturingCommandLine;
+import org.jabref.toolkit.util.CommandFactory;
+
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+import picocli.CommandLine;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+class GetFulltextsTest extends AbstractJabKitTest {
+
+    @TempDir
+    private Path tempDir;
+
+    private final FulltextDownloader downloader = mock(FulltextDownloader.class);
+
+    @BeforeEach
+    void setupDownloader() {
+        GetFulltexts sut = new GetFulltexts() {
+            @Override
+            void initFields() {
+                this.fulltextDownloader = downloader;
+            }
+        };
+        commandLine = new CapturingCommandLine(new JabKit(preferences, entryTypesManager), new CommandFactory(sut));
+    }
+
+    @Test
+    void updatesLibraryInPlaceAndReportsEachEntry() throws IOException {
+        Path library = tempDir.resolve("library.bib");
+        Files.writeString(library, """
+                @Article{Found,
+                }
+                @Article{Missing,
+                }
+                @Article{HasPdf,
+                  file = {:HasPdf.pdf:PDF},
+                }
+                """);
+        Path downloadedFile = tempDir.resolve("Found.pdf");
+        when(downloader.download(any(), any())).thenAnswer(invocation -> {
+            BibEntry entry = invocation.getArgument(1);
+            if ("Found".equals(entry.getCitationKey().orElseThrow())) {
+                entry.addFile(new LinkedFile("", Path.of("Found.pdf"), "PDF"));
+                return new FulltextDownloader.Result.Downloaded(downloadedFile);
+            }
+            return new FulltextDownloader.Result.NotFound();
+        });
+
+        int exitCode = commandLine.executeToLog("get-fulltexts", library.toString());
+
+        assertEquals(CommandLine.ExitCode.OK, exitCode);
+        String output = commandLine.getStandardOutput();
+        assertTrue(output.contains("Downloaded full text document for entry Found: " + downloadedFile), output);
+        assertTrue(output.contains("No full text document found for entry Missing."), output);
+        assertTrue(output.contains("Full text document for entry HasPdf already linked."), output);
+        assertTrue(output.contains("Full text documents: 1 downloaded, 1 not found, 1 skipped, 0 failed."), output);
+        verify(downloader, times(2)).download(any(), any());
+        assertTrue(Files.readString(library).contains(":Found.pdf:PDF"), Files.readString(library));
+    }
+}

jablib/src/main/java/org/jabref/logic/externalfiles/FulltextDownloader.java

@@ -0,0 +1,120 @@
+package org.jabref.logic.externalfiles;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Optional;
+import java.util.function.Function;
+
+import org.jabref.logic.FilePreferences;
+import org.jabref.logic.importer.FetcherException;
+import org.jabref.logic.importer.FetcherResult;
+import org.jabref.logic.importer.FulltextFetchers;
+import org.jabref.logic.importer.ImportFormatPreferences;
+import org.jabref.logic.importer.ImporterPreferences;
+import org.jabref.logic.net.URLDownload;
+import org.jabref.logic.util.StandardFileType;
+import org.jabref.logic.util.io.FileNameUniqueness;
+import org.jabref.logic.util.io.FileUtil;
+import org.jabref.model.database.BibDatabaseContext;
+import org.jabref.model.entry.BibEntry;
+import org.jabref.model.entry.LinkedFile;
+
+import org.jspecify.annotations.NullMarked;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/// Searches the full text PDF of an entry online, downloads it into the library's file directory and links it.
+///
+/// UI-independent, so that `jabkit get-fulltexts` can use it; the GUI's "Search full text documents online" still has
+/// its own implementation.
+@NullMarked
+public class FulltextDownloader {
+
+    private static final Logger LOGGER = LoggerFactory.getLogger(FulltextDownloader.class);
+
+    public sealed interface Result {
+        /// @param file the downloaded file, absolute
+        record Downloaded(Path file) implements Result {
+        }
+
+        record NotFound() implements Result {
+        }
+
+        record AlreadyLinked() implements Result {
+        }
+
+        /// The downloaded file had the same content as an existing file in the target directory and was deleted again.
+        record Duplicate() implements Result {
+        }
+
+        record NoFileDirectory() implements Result {
+        }
+
+        record Failed(String message) implements Result {
+        }
+    }
+
+    private final FilePreferences filePreferences;
+    private final Function<BibEntry, Optional<FetcherResult>> fullTextFinder;
+
+    public FulltextDownloader(ImportFormatPreferences importFormatPreferences,
+                              ImporterPreferences importerPreferences,
+                              FilePreferences filePreferences) {
+        this(filePreferences, new FulltextFetchers(importFormatPreferences, importerPreferences)::findFullTextPDF);
+    }
+
+    FulltextDownloader(FilePreferences filePreferences, Function<BibEntry, Optional<FetcherResult>> fullTextFinder) {
+        this.filePreferences = filePreferences;
+        this.fullTextFinder = fullTextFinder;
+    }
+
+    /// Links the downloaded file to the entry; the caller has to save the library.
+    public Result download(BibDatabaseContext databaseContext, BibEntry entry) {
+        return databaseContext.getFirstExistingFileDir(filePreferences)
+                              .map(targetDirectory -> fullTextFinder.apply(entry)
+                                                                    .map(found -> download(databaseContext, entry, found, targetDirectory))
+                                                                    .orElseGet(Result.NotFound::new))
+                              .orElseGet(Result.NoFileDirectory::new);
+    }
+
+    private Result download(BibDatabaseContext databaseContext, BibEntry entry, FetcherResult found, Path targetDirectory) {
+        String url = found.source().toExternalForm();
+        if (entry.getFiles().stream().anyMatch(file -> url.equals(file.getLink()) || url.equals(file.getSourceUrl()))) {
+            return new Result.AlreadyLinked();
+        }
+
+        String fileName = new LinkedFileHandler(new LinkedFile(found.source(), ""), entry, databaseContext, filePreferences)
+                .getSuggestedFileName("pdf");
+        Path directory = targetDirectory.resolve(FileUtil.createDirNameFromPattern(databaseContext.getDatabase(), entry, filePreferences.getFileDirectoryPattern()));
+        Path destination = directory.resolve(FileNameUniqueness.getNonOverWritingFileName(directory, fileName));
+        try {
+            Files.createDirectories(directory);
+            URLDownload download = new URLDownload(found.source());
+            found.headers().forEach(download::addHeader);
+            download.toFile(destination);
+            if (FileNameUniqueness.isDuplicatedFile(directory, destination.getFileName(), LOGGER::info)) {
+                return new Result.Duplicate();
+            }
+        } catch (IOException | FetcherException e) {
+            LOGGER.warn("Could not download {}", FetcherException.getRedactedUrl(url), e);
+            deletePartialDownload(destination);
+            return new Result.Failed(e.getLocalizedMessage());
+        }
+
+        LinkedFile linkedFile = new LinkedFile("", FileUtil.relativize(destination, databaseContext.getFileDirectories(filePreferences)), StandardFileType.PDF.getName());
+        if (filePreferences.shouldKeepDownloadUrl()) {
+            linkedFile.setSourceURL(url);
+        }
+        entry.addFile(linkedFile);
+        return new Result.Downloaded(destination);
+    }
+
+    private static void deletePartialDownload(Path destination) {
+        try {
+            Files.deleteIfExists(destination);
+        } catch (IOException e) {
+            LOGGER.warn("Could not delete partially downloaded {}", destination, e);
+        }
+    }
+}

jablib/src/main/resources/l10n/JabRef_en.properties

@@ -1962,6 +1962,11 @@ Opens\ a\ link\ where\ the\ current\ development\ version\ can\ be\ downloaded=O
 See\ what\ has\ been\ changed\ in\ the\ JabRef\ versions=See what has been changed in the JabRef versions
 Referenced\ citation\ key\ '%0'\ does\ not\ exist=Referenced citation key '%0' does not exist
 Full\ text\ document\ for\ entry\ %0\ already\ linked.=Full text document for entry %0 already linked.
+Downloaded\ full\ text\ document\ for\ entry\ %0\:\ %1=Downloaded full text document for entry %0: %1
+Full\ text\ document\ for\ entry\ %0\ is\ a\ duplicate\ of\ an\ existing\ file.=Full text document for entry %0 is a duplicate of an existing file.
+Could\ not\ download\ the\ full\ text\ document\ for\ entry\ %0\:\ %1=Could not download the full text document for entry %0: %1
+No\ existing\ file\ directory\ to\ download\ the\ full\ text\ documents\ to.=No existing file directory to download the full text documents to.
+Full\ text\ documents\:\ %0\ downloaded,\ %1\ not\ found,\ %2\ skipped,\ %3\ failed.=Full text documents: %0 downloaded, %1 not found, %2 skipped, %3 failed.
 Download\ full\ text\ documents=Download full text documents
 You\ are\ attempting\ to\ download\ full\ text\ documents\ for\ %0\ entries.\nJabRef\ will\ send\ at\ least\ one\ request\ per\ entry\ to\ a\ publisher.=You are attempting to download full text documents for %0 entries.\nJabRef will send at least one request per entry to a publisher.
 last\ four\ nonpunctuation\ characters\ should\ be\ numerals=last four nonpunctuation characters should be numerals