From bfac1cb40bcf8a2aff1d591ad5cf92200543df49 Mon Sep 17 00:00:00 2001 From: Oliver Kopp Date: Tue, 22 Sep 2026 13:14:33 +0200 Subject: [PATCH 1/5] Add jabkit get-fulltext Downloads and links the full text PDFs of a library's entries from the command line, printing one line per entry and a summary. Co-Authored-By: Claude Opus 5 --- .jbang/JabKitLauncher.java | 1 + docs/requirements/cli.md | 10 ++ .../jabref/toolkit/commands/GetFulltext.java | 115 ++++++++++++++ .../org/jabref/toolkit/commands/JabKit.java | 1 + .../toolkit/commands/GetFulltextTest.java | 77 ++++++++++ .../externalfiles/FulltextDownloader.java | 122 +++++++++++++++ .../main/resources/l10n/JabRef_en.properties | 5 + .../externalfiles/FulltextDownloaderTest.java | 140 ++++++++++++++++++ skills/users/jabkit/SKILL.md | 4 + 9 files changed, 475 insertions(+) create mode 100644 jabkit/src/main/java/org/jabref/toolkit/commands/GetFulltext.java create mode 100644 jabkit/src/test/java/org/jabref/toolkit/commands/GetFulltextTest.java create mode 100644 jablib/src/main/java/org/jabref/logic/externalfiles/FulltextDownloader.java create mode 100644 jablib/src/test/java/org/jabref/logic/externalfiles/FulltextDownloaderTest.java diff --git a/.jbang/JabKitLauncher.java b/.jbang/JabKitLauncher.java index 12cdd8a0ab67..b825d7b8bd12 100755 --- a/.jbang/JabKitLauncher.java +++ b/.jbang/JabKitLauncher.java @@ -32,6 +32,7 @@ //SOURCES ../jabkit/src/main/java/org/jabref/toolkit/commands/GenerateCitationKeys.java //SOURCES ../jabkit/src/main/java/org/jabref/toolkit/commands/GetCitedWorks.java //SOURCES ../jabkit/src/main/java/org/jabref/toolkit/commands/GetCitingWorks.java +//SOURCES ../jabkit/src/main/java/org/jabref/toolkit/commands/GetFulltext.java //SOURCES ../jabkit/src/main/java/org/jabref/toolkit/commands/Git.java //SOURCES ../jabkit/src/main/java/org/jabref/toolkit/commands/GitMergeDriver.java //SOURCES ../jabkit/src/main/java/org/jabref/toolkit/commands/InputOption.java diff --git a/docs/requirements/cli.md b/docs/requirements/cli.md index a36b2098c218..2ef90acdf62a 100644 --- a/docs/requirements/cli.md +++ b/docs/requirements/cli.md @@ -93,4 +93,14 @@ are parsed correctly by the GitHub Actions runner. Needs: impl +## Full text download reports each entry +`req~jabkit.cli.get-fulltext-report~1` + +`jabkit get-fulltext` updates the given library in place (or writes to `--output`). +It prints one line per entry saying whether its full text document was downloaded, not found, +skipped, or failed, followed by a summary with the counts. +Entries that already link a local PDF are skipped without contacting a publisher. + +Needs: impl + diff --git a/jabkit/src/main/java/org/jabref/toolkit/commands/GetFulltext.java b/jabkit/src/main/java/org/jabref/toolkit/commands/GetFulltext.java new file mode 100644 index 000000000000..6630f0111060 --- /dev/null +++ b/jabkit/src/main/java/org/jabref/toolkit/commands/GetFulltext.java @@ -0,0 +1,115 @@ +package org.jabref.toolkit.commands; + +import java.nio.file.Path; +import java.util.concurrent.Callable; + +import org.jabref.logic.externalfiles.FulltextDownloader; +import org.jabref.logic.importer.ParserResult; +import org.jabref.logic.l10n.Localization; +import org.jabref.logic.preferences.CliPreferences; +import org.jabref.logic.util.StandardFileType; +import org.jabref.model.database.BibDatabaseContext; +import org.jabref.model.entry.BibEntry; +import org.jabref.model.entry.LinkedFile; +import org.jabref.toolkit.exception.ExportServiceException; +import org.jabref.toolkit.exception.ImportServiceException; +import org.jabref.toolkit.service.ExportService; +import org.jabref.toolkit.service.ImportService; + +import picocli.CommandLine; +import picocli.CommandLine.Command; +import picocli.CommandLine.Mixin; +import picocli.CommandLine.Option; +import picocli.CommandLine.ParentCommand; + +// [impl->req~jabkit.cli.get-fulltext-report~1] +@Command(name = "get-fulltext", description = "Download the full text PDFs of the entries of a library and link them.") +class GetFulltext implements Callable { + + protected FulltextDownloader fulltextDownloader; + + @ParentCommand + private JabKit argumentProcessor; + + @Mixin + private JabKit.SharedOptions sharedOptions; + + @Mixin + private InputOption inputOption = new InputOption(); + + @Option(names = "--output", description = "Output .bib file (default: the input file is updated)") + private Path outputFile; + + void initFields() { + CliPreferences preferences = argumentProcessor.cliPreferences; + fulltextDownloader = new FulltextDownloader(preferences.getImportFormatPreferences(), preferences.getImporterPreferences(), preferences.getFilePreferences()); + } + + @Override + public Integer call() throws ImportServiceException, ExportServiceException { + initFields(); + CliPreferences preferences = argumentProcessor.cliPreferences; + Path inputFile = inputOption.getInputFile(preferences); + ParserResult parserResult = ImportService.importBibTexFile(inputFile, preferences, sharedOptions.porcelain); + BibDatabaseContext databaseContext = parserResult.getDatabaseContext(); + // Relative file directories and the "store files next to the library" preference resolve against it + databaseContext.setDatabasePath(inputFile.toAbsolutePath()); + + int downloaded = 0; + int notFound = 0; + int skipped = 0; + int failed = 0; + for (BibEntry entry : databaseContext.getEntries()) { + String citationKey = entry.getCitationKey().orElse(Localization.lang("undefined")); + // Keeps a re-run from sending requests to publishers for entries it already handled + if (hasLocalPdf(entry)) { + System.out.println(Localization.lang("Full text document for entry %0 already linked.", citationKey)); + skipped++; + continue; + } + switch (fulltextDownloader.download(databaseContext, entry)) { + case FulltextDownloader.Result.Downloaded downloadedFile -> { + System.out.println(Localization.lang("Downloaded full text document for entry %0: %1", citationKey, downloadedFile.file())); + downloaded++; + } + case FulltextDownloader.Result.NotFound _ -> { + System.out.println(Localization.lang("No full text document found for entry %0.", citationKey)); + notFound++; + } + case FulltextDownloader.Result.AlreadyLinked _ -> { + System.out.println(Localization.lang("Full text document for entry %0 already linked.", citationKey)); + skipped++; + } + case FulltextDownloader.Result.Duplicate _ -> { + System.out.println(Localization.lang("Full text document for entry %0 is a duplicate of an existing file.", citationKey)); + skipped++; + } + case FulltextDownloader.Result.Failed failure -> { + System.out.println(Localization.lang("Could not download the full text document for entry %0: %1", citationKey, failure.message())); + failed++; + } + case FulltextDownloader.Result.NoFileDirectory _ -> { + System.err.println(Localization.lang("No existing file directory to download the full text documents to.")); + return CommandLine.ExitCode.SOFTWARE; + } + } + } + + if (!sharedOptions.porcelain) { + System.out.println(Localization.lang("Full text documents: %0 downloaded, %1 not found, %2 skipped, %3 failed.", + downloaded, notFound, skipped, failed)); + } + + if (downloaded > 0 || outputFile != null) { + ExportService.create(preferences, sharedOptions.porcelain) + .saveDatabaseContext(databaseContext, outputFile != null ? outputFile : inputFile); + } + return failed > 0 ? CommandLine.ExitCode.SOFTWARE : CommandLine.ExitCode.OK; + } + + private static boolean hasLocalPdf(BibEntry entry) { + return entry.getFiles().stream() + .anyMatch(file -> !LinkedFile.isOnlineLink(file.getLink()) + && StandardFileType.PDF.getName().equalsIgnoreCase(file.getFileType())); + } +} diff --git a/jabkit/src/main/java/org/jabref/toolkit/commands/JabKit.java b/jabkit/src/main/java/org/jabref/toolkit/commands/JabKit.java index d605a0d81c30..51707ff69727 100644 --- a/jabkit/src/main/java/org/jabref/toolkit/commands/JabKit.java +++ b/jabkit/src/main/java/org/jabref/toolkit/commands/JabKit.java @@ -23,6 +23,7 @@ GenerateBibFromAux.class, GetCitedWorks.class, GetCitingWorks.class, + GetFulltext.class, Git.class, Pdf.class, Preferences.class, diff --git a/jabkit/src/test/java/org/jabref/toolkit/commands/GetFulltextTest.java b/jabkit/src/test/java/org/jabref/toolkit/commands/GetFulltextTest.java new file mode 100644 index 000000000000..320031a19345 --- /dev/null +++ b/jabkit/src/test/java/org/jabref/toolkit/commands/GetFulltextTest.java @@ -0,0 +1,77 @@ +package org.jabref.toolkit.commands; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; + +import org.jabref.logic.externalfiles.FulltextDownloader; +import org.jabref.model.entry.BibEntry; +import org.jabref.model.entry.LinkedFile; +import org.jabref.toolkit.util.CapturingCommandLine; +import org.jabref.toolkit.util.CommandFactory; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import picocli.CommandLine; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +class GetFulltextTest extends AbstractJabKitTest { + + @TempDir + private Path tempDir; + + private final FulltextDownloader downloader = mock(FulltextDownloader.class); + + @BeforeEach + void setupDownloader() { + GetFulltext sut = new GetFulltext() { + @Override + void initFields() { + this.fulltextDownloader = downloader; + } + }; + commandLine = new CapturingCommandLine(new JabKit(preferences, entryTypesManager), new CommandFactory(sut)); + } + + @Test + void updatesLibraryInPlaceAndReportsEachEntry() throws IOException { + Path library = tempDir.resolve("library.bib"); + Files.writeString(library, """ + @Article{Found, + } + @Article{Missing, + } + @Article{HasPdf, + file = {:HasPdf.pdf:PDF}, + } + """); + Path downloadedFile = tempDir.resolve("Found.pdf"); + when(downloader.download(any(), any())).thenAnswer(invocation -> { + BibEntry entry = invocation.getArgument(1); + if (entry.getCitationKey().orElseThrow().equals("Found")) { + entry.addFile(new LinkedFile("", Path.of("Found.pdf"), "PDF")); + return new FulltextDownloader.Result.Downloaded(downloadedFile); + } + return new FulltextDownloader.Result.NotFound(); + }); + + int exitCode = commandLine.executeToLog("get-fulltext", library.toString()); + + assertEquals(CommandLine.ExitCode.OK, exitCode); + String output = commandLine.getStandardOutput(); + assertTrue(output.contains("Downloaded full text document for entry Found: " + downloadedFile), output); + assertTrue(output.contains("No full text document found for entry Missing."), output); + assertTrue(output.contains("Full text document for entry HasPdf already linked."), output); + assertTrue(output.contains("Full text documents: 1 downloaded, 1 not found, 1 skipped, 0 failed."), output); + verify(downloader, times(2)).download(any(), any()); + assertTrue(Files.readString(library).contains(":Found.pdf:PDF"), Files.readString(library)); + } +} diff --git a/jablib/src/main/java/org/jabref/logic/externalfiles/FulltextDownloader.java b/jablib/src/main/java/org/jabref/logic/externalfiles/FulltextDownloader.java new file mode 100644 index 000000000000..41b6a0ee66bf --- /dev/null +++ b/jablib/src/main/java/org/jabref/logic/externalfiles/FulltextDownloader.java @@ -0,0 +1,122 @@ +package org.jabref.logic.externalfiles; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Optional; +import java.util.function.Function; + +import org.jabref.logic.FilePreferences; +import org.jabref.logic.importer.FetcherException; +import org.jabref.logic.importer.FetcherResult; +import org.jabref.logic.importer.FulltextFetchers; +import org.jabref.logic.importer.ImportFormatPreferences; +import org.jabref.logic.importer.ImporterPreferences; +import org.jabref.logic.net.URLDownload; +import org.jabref.logic.util.StandardFileType; +import org.jabref.logic.util.io.FileNameUniqueness; +import org.jabref.logic.util.io.FileUtil; +import org.jabref.model.database.BibDatabaseContext; +import org.jabref.model.entry.BibEntry; +import org.jabref.model.entry.LinkedFile; + +import org.jspecify.annotations.NullMarked; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/// Searches the full text PDF of an entry online, downloads it into the library's file directory and links it. +/// +/// UI-independent, so that `jabkit get-fulltext` can use it; the GUI's "Search full text documents online" still has +/// its own implementation. +@NullMarked +public class FulltextDownloader { + + private static final Logger LOGGER = LoggerFactory.getLogger(FulltextDownloader.class); + + public sealed interface Result { + /// @param file the downloaded file, absolute + record Downloaded(Path file) implements Result { + } + + record NotFound() implements Result { + } + + record AlreadyLinked() implements Result { + } + + /// The downloaded file had the same content as an existing file in the target directory and was deleted again. + record Duplicate() implements Result { + } + + record NoFileDirectory() implements Result { + } + + record Failed(String message) implements Result { + } + } + + private final FilePreferences filePreferences; + private final Function> fullTextFinder; + + public FulltextDownloader(ImportFormatPreferences importFormatPreferences, + ImporterPreferences importerPreferences, + FilePreferences filePreferences) { + this(filePreferences, new FulltextFetchers(importFormatPreferences, importerPreferences)::findFullTextPDF); + } + + FulltextDownloader(FilePreferences filePreferences, Function> fullTextFinder) { + this.filePreferences = filePreferences; + this.fullTextFinder = fullTextFinder; + } + + /// Links the downloaded file to the entry; the caller has to save the library. + public Result download(BibDatabaseContext databaseContext, BibEntry entry) { + Optional targetDirectory = databaseContext.getFirstExistingFileDir(filePreferences); + if (targetDirectory.isEmpty()) { + return new Result.NoFileDirectory(); + } + + Optional fetcherResult = fullTextFinder.apply(entry); + if (fetcherResult.isEmpty()) { + return new Result.NotFound(); + } + + String url = fetcherResult.get().source().toExternalForm(); + if (entry.getFiles().stream().anyMatch(file -> url.equals(file.getLink()) || url.equals(file.getSourceUrl()))) { + return new Result.AlreadyLinked(); + } + + String fileName = new LinkedFileHandler(new LinkedFile(fetcherResult.get().source(), ""), entry, databaseContext, filePreferences) + .getSuggestedFileName("pdf"); + Path directory = targetDirectory.get().resolve(FileUtil.createDirNameFromPattern(databaseContext.getDatabase(), entry, filePreferences.getFileDirectoryPattern())); + Path destination = directory.resolve(FileNameUniqueness.getNonOverWritingFileName(directory, fileName)); + try { + Files.createDirectories(directory); + URLDownload download = new URLDownload(fetcherResult.get().source()); + fetcherResult.get().headers().forEach(download::addHeader); + download.toFile(destination); + if (FileNameUniqueness.isDuplicatedFile(directory, destination.getFileName(), LOGGER::info)) { + return new Result.Duplicate(); + } + } catch (IOException | FetcherException e) { + LOGGER.warn("Could not download {}", FetcherException.getRedactedUrl(url), e); + deletePartialDownload(destination); + return new Result.Failed(e.getLocalizedMessage()); + } + + LinkedFile linkedFile = new LinkedFile("", FileUtil.relativize(destination, databaseContext.getFileDirectories(filePreferences)), StandardFileType.PDF.getName()); + if (filePreferences.shouldKeepDownloadUrl()) { + linkedFile.setSourceURL(url); + } + entry.addFile(linkedFile); + return new Result.Downloaded(destination); + } + + private static void deletePartialDownload(Path destination) { + try { + Files.deleteIfExists(destination); + } catch (IOException e) { + LOGGER.warn("Could not delete partially downloaded {}", destination, e); + } + } +} diff --git a/jablib/src/main/resources/l10n/JabRef_en.properties b/jablib/src/main/resources/l10n/JabRef_en.properties index 63c933b56930..3a74192235c0 100644 --- a/jablib/src/main/resources/l10n/JabRef_en.properties +++ b/jablib/src/main/resources/l10n/JabRef_en.properties @@ -1962,6 +1962,11 @@ Opens\ a\ link\ where\ the\ current\ development\ version\ can\ be\ downloaded=O See\ what\ has\ been\ changed\ in\ the\ JabRef\ versions=See what has been changed in the JabRef versions Referenced\ citation\ key\ '%0'\ does\ not\ exist=Referenced citation key '%0' does not exist Full\ text\ document\ for\ entry\ %0\ already\ linked.=Full text document for entry %0 already linked. +Downloaded\ full\ text\ document\ for\ entry\ %0\:\ %1=Downloaded full text document for entry %0: %1 +Full\ text\ document\ for\ entry\ %0\ is\ a\ duplicate\ of\ an\ existing\ file.=Full text document for entry %0 is a duplicate of an existing file. +Could\ not\ download\ the\ full\ text\ document\ for\ entry\ %0\:\ %1=Could not download the full text document for entry %0: %1 +No\ existing\ file\ directory\ to\ download\ the\ full\ text\ documents\ to.=No existing file directory to download the full text documents to. +Full\ text\ documents\:\ %0\ downloaded,\ %1\ not\ found,\ %2\ skipped,\ %3\ failed.=Full text documents: %0 downloaded, %1 not found, %2 skipped, %3 failed. Download\ full\ text\ documents=Download full text documents You\ are\ attempting\ to\ download\ full\ text\ documents\ for\ %0\ entries.\nJabRef\ will\ send\ at\ least\ one\ request\ per\ entry\ to\ a\ publisher.=You are attempting to download full text documents for %0 entries.\nJabRef will send at least one request per entry to a publisher. last\ four\ nonpunctuation\ characters\ should\ be\ numerals=last four nonpunctuation characters should be numerals diff --git a/jablib/src/test/java/org/jabref/logic/externalfiles/FulltextDownloaderTest.java b/jablib/src/test/java/org/jabref/logic/externalfiles/FulltextDownloaderTest.java new file mode 100644 index 000000000000..ee783c9f9d7b --- /dev/null +++ b/jablib/src/test/java/org/jabref/logic/externalfiles/FulltextDownloaderTest.java @@ -0,0 +1,140 @@ +package org.jabref.logic.externalfiles; + +import java.io.IOException; +import java.io.OutputStream; +import java.net.InetSocketAddress; +import java.net.URI; +import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.Optional; +import java.util.stream.Stream; + +import org.jabref.logic.FilePreferences; +import org.jabref.logic.importer.FetcherResult; +import org.jabref.logic.importer.fetcher.TrustLevel; +import org.jabref.model.database.BibDatabase; +import org.jabref.model.database.BibDatabaseContext; +import org.jabref.model.entry.BibEntry; +import org.jabref.model.entry.LinkedFile; + +import com.sun.net.httpserver.HttpServer; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +class FulltextDownloaderTest { + + private static final byte[] PDF_CONTENT = "%PDF-1.4 fake".getBytes(StandardCharsets.US_ASCII); + + @TempDir + private Path libraryDirectory; + + private final FilePreferences filePreferences = mock(FilePreferences.class); + private HttpServer server; + private URL pdfUrl; + private BibEntry entry; + private BibDatabaseContext databaseContext; + + @BeforeEach + void setUp() throws IOException { + server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0); + server.createContext("/paper.pdf", exchange -> { + exchange.getResponseHeaders().add("Content-Type", "application/pdf"); + exchange.sendResponseHeaders(200, PDF_CONTENT.length); + try (OutputStream body = exchange.getResponseBody()) { + body.write(PDF_CONTENT); + } + }); + server.start(); + pdfUrl = URI.create("http://127.0.0.1:" + server.getAddress().getPort() + "/paper.pdf").toURL(); + + when(filePreferences.shouldStoreFilesRelativeToBibFile()).thenReturn(true); + when(filePreferences.getFileNamePattern()).thenReturn("[citationkey]"); + when(filePreferences.getFileDirectoryPattern()).thenReturn(""); + + entry = new BibEntry().withCitationKey("Tan_2021"); + databaseContext = new BibDatabaseContext(new BibDatabase(List.of(entry))); + databaseContext.setDatabasePath(libraryDirectory.resolve("library.bib")); + } + + @AfterEach + void tearDown() { + server.stop(0); + } + + private List filesInLibraryDirectory() throws IOException { + try (Stream files = Files.list(libraryDirectory)) { + return files.toList(); + } + } + + private FulltextDownloader downloaderFinding(Optional url) { + return new FulltextDownloader(filePreferences, _ -> url.map(found -> new FetcherResult(TrustLevel.PUBLISHER, found))); + } + + @Test + void downloadsAndLinksPdf() throws IOException { + Path expectedFile = libraryDirectory.resolve("Tan_2021.pdf"); + + FulltextDownloader.Result result = downloaderFinding(Optional.of(pdfUrl)).download(databaseContext, entry); + + assertEquals(new FulltextDownloader.Result.Downloaded(expectedFile), result); + assertEquals(List.of(new LinkedFile("", Path.of("Tan_2021.pdf"), "PDF")), entry.getFiles()); + assertArrayEquals(PDF_CONTENT, Files.readAllBytes(expectedFile)); + } + + @Test + void identicalExistingFileIsNotDownloadedTwice() throws IOException { + Files.write(libraryDirectory.resolve("Tan_2021.pdf"), PDF_CONTENT); + + FulltextDownloader.Result result = downloaderFinding(Optional.of(pdfUrl)).download(databaseContext, entry); + + assertEquals(new FulltextDownloader.Result.Duplicate(), result); + assertEquals(List.of(), entry.getFiles()); + assertEquals(List.of(libraryDirectory.resolve("Tan_2021.pdf")), filesInLibraryDirectory()); + } + + @Test + void reportsNotFound() { + FulltextDownloader.Result result = downloaderFinding(Optional.empty()).download(databaseContext, entry); + + assertEquals(new FulltextDownloader.Result.NotFound(), result); + } + + @Test + void reportsAlreadyLinkedUrl() { + entry.addFile(new LinkedFile("", pdfUrl, "")); + + FulltextDownloader.Result result = downloaderFinding(Optional.of(pdfUrl)).download(databaseContext, entry); + + assertEquals(new FulltextDownloader.Result.AlreadyLinked(), result); + } + + @Test + void reportsMissingFileDirectory() { + databaseContext.setDatabasePath(libraryDirectory.resolve("missing").resolve("library.bib")); + + FulltextDownloader.Result result = downloaderFinding(Optional.of(pdfUrl)).download(databaseContext, entry); + + assertEquals(new FulltextDownloader.Result.NoFileDirectory(), result); + } + + @Test + void failedDownloadLeavesNoFile() throws IOException { + URL missing = URI.create("http://127.0.0.1:" + server.getAddress().getPort() + "/missing.pdf").toURL(); + + FulltextDownloader.Result result = downloaderFinding(Optional.of(missing)).download(databaseContext, entry); + + assertEquals(FulltextDownloader.Result.Failed.class, result.getClass()); + assertEquals(List.of(), filesInLibraryDirectory()); + } +} diff --git a/skills/users/jabkit/SKILL.md b/skills/users/jabkit/SKILL.md index d7bc2630b8a2..7a1a171de1ae 100644 --- a/skills/users/jabkit/SKILL.md +++ b/skills/users/jabkit/SKILL.md @@ -89,6 +89,7 @@ Place before the subcommand: | `generate-bib-from-aux` | Extract the subset of a library cited in a LaTeX `.aux` file | | `get-cited-works DOI` | List the works cited by a publication | | `get-citing-works DOI` | List the works citing a publication | +| `get-fulltext FILE` | Download the full text PDFs of a library's entries and link them (updates FILE in place) | | `pdf extract-references FILE...` | Parse the "References" section of PDFs into BibTeX entries | | `pdf update` | Write XMP metadata and/or embedded BibTeX into linked PDFs | | `preferences reset\|import\|export` | Manage jabkit preferences | @@ -121,6 +122,9 @@ jabkit -p pdf extract-references paper.pdf # Write BibTeX + XMP metadata into the PDFs linked from an entry jabkit pdf update --citation-key Smith2020 --input library.bib --input-format bibtex +# Download and link the full text PDFs (entries with a linked PDF are skipped) +jabkit get-fulltext library.bib + # Library subset actually cited in a LaTeX document jabkit generate-bib-from-aux --aux paper.aux --input full-library.bib --output paper.bib ``` From 94e09404256cbbc1f0341ede533c948f772ddbe9 Mon Sep 17 00:00:00 2001 From: Oliver Kopp Date: Tue, 22 Sep 2026 13:31:11 +0200 Subject: [PATCH 2/5] Add changelog entry, avoid Optional get Co-Authored-By: Claude Opus 5 --- CHANGELOG.md | 1 + .../toolkit/commands/GetFulltextTest.java | 2 +- .../externalfiles/FulltextDownloader.java | 26 +++++++++---------- 3 files changed, 14 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e2da4e5c6734..60f842a06d3f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ Note that this project **does not** adhere to [Semantic Versioning](https://semv ### Added - We added `jabkit git merge-driver`, a Git merge driver that merges `.bib` files semantically. [#16838](https://github.com/JabRef/jabref/pull/16838) +- We added `jabkit get-fulltext` to download and link full text PDFs, reporting the result per entry. TODO ### Changed diff --git a/jabkit/src/test/java/org/jabref/toolkit/commands/GetFulltextTest.java b/jabkit/src/test/java/org/jabref/toolkit/commands/GetFulltextTest.java index 320031a19345..b7d00a5ef0f8 100644 --- a/jabkit/src/test/java/org/jabref/toolkit/commands/GetFulltextTest.java +++ b/jabkit/src/test/java/org/jabref/toolkit/commands/GetFulltextTest.java @@ -56,7 +56,7 @@ void updatesLibraryInPlaceAndReportsEachEntry() throws IOException { Path downloadedFile = tempDir.resolve("Found.pdf"); when(downloader.download(any(), any())).thenAnswer(invocation -> { BibEntry entry = invocation.getArgument(1); - if (entry.getCitationKey().orElseThrow().equals("Found")) { + if ("Found".equals(entry.getCitationKey().orElseThrow())) { entry.addFile(new LinkedFile("", Path.of("Found.pdf"), "PDF")); return new FulltextDownloader.Result.Downloaded(downloadedFile); } diff --git a/jablib/src/main/java/org/jabref/logic/externalfiles/FulltextDownloader.java b/jablib/src/main/java/org/jabref/logic/externalfiles/FulltextDownloader.java index 41b6a0ee66bf..2e32e0559a60 100644 --- a/jablib/src/main/java/org/jabref/logic/externalfiles/FulltextDownloader.java +++ b/jablib/src/main/java/org/jabref/logic/externalfiles/FulltextDownloader.java @@ -71,29 +71,27 @@ public FulltextDownloader(ImportFormatPreferences importFormatPreferences, /// Links the downloaded file to the entry; the caller has to save the library. public Result download(BibDatabaseContext databaseContext, BibEntry entry) { - Optional targetDirectory = databaseContext.getFirstExistingFileDir(filePreferences); - if (targetDirectory.isEmpty()) { - return new Result.NoFileDirectory(); - } - - Optional fetcherResult = fullTextFinder.apply(entry); - if (fetcherResult.isEmpty()) { - return new Result.NotFound(); - } + return databaseContext.getFirstExistingFileDir(filePreferences) + .map(targetDirectory -> fullTextFinder.apply(entry) + .map(found -> download(databaseContext, entry, found, targetDirectory)) + .orElseGet(Result.NotFound::new)) + .orElseGet(Result.NoFileDirectory::new); + } - String url = fetcherResult.get().source().toExternalForm(); + private Result download(BibDatabaseContext databaseContext, BibEntry entry, FetcherResult found, Path targetDirectory) { + String url = found.source().toExternalForm(); if (entry.getFiles().stream().anyMatch(file -> url.equals(file.getLink()) || url.equals(file.getSourceUrl()))) { return new Result.AlreadyLinked(); } - String fileName = new LinkedFileHandler(new LinkedFile(fetcherResult.get().source(), ""), entry, databaseContext, filePreferences) + String fileName = new LinkedFileHandler(new LinkedFile(found.source(), ""), entry, databaseContext, filePreferences) .getSuggestedFileName("pdf"); - Path directory = targetDirectory.get().resolve(FileUtil.createDirNameFromPattern(databaseContext.getDatabase(), entry, filePreferences.getFileDirectoryPattern())); + Path directory = targetDirectory.resolve(FileUtil.createDirNameFromPattern(databaseContext.getDatabase(), entry, filePreferences.getFileDirectoryPattern())); Path destination = directory.resolve(FileNameUniqueness.getNonOverWritingFileName(directory, fileName)); try { Files.createDirectories(directory); - URLDownload download = new URLDownload(fetcherResult.get().source()); - fetcherResult.get().headers().forEach(download::addHeader); + URLDownload download = new URLDownload(found.source()); + found.headers().forEach(download::addHeader); download.toFile(destination); if (FileNameUniqueness.isDuplicatedFile(directory, destination.getFileName(), LOGGER::info)) { return new Result.Duplicate(); From 53552f4f73e5622c8cc577d6507f087e9bb32dad Mon Sep 17 00:00:00 2001 From: Oliver Kopp Date: Tue, 22 Sep 2026 13:31:55 +0200 Subject: [PATCH 3/5] Link PR in changelog Co-Authored-By: Claude Opus 5 --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 60f842a06d3f..0eb4dedae06a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,7 +12,7 @@ Note that this project **does not** adhere to [Semantic Versioning](https://semv ### Added - We added `jabkit git merge-driver`, a Git merge driver that merges `.bib` files semantically. [#16838](https://github.com/JabRef/jabref/pull/16838) -- We added `jabkit get-fulltext` to download and link full text PDFs, reporting the result per entry. TODO +- We added `jabkit get-fulltext` to download and link full text PDFs, reporting the result per entry. [koppor#770](https://github.com/JabRef/jabref-koppor/pull/770) ### Changed From 8b8cb939a9ec8e478998509efc0964ceb93d9a02 Mon Sep 17 00:00:00 2001 From: Oliver Kopp Date: Tue, 22 Sep 2026 13:32:44 +0200 Subject: [PATCH 4/5] Use #770 as changelog link text Co-Authored-By: Claude Opus 5 --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0eb4dedae06a..8093682e7edf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,7 +12,7 @@ Note that this project **does not** adhere to [Semantic Versioning](https://semv ### Added - We added `jabkit git merge-driver`, a Git merge driver that merges `.bib` files semantically. [#16838](https://github.com/JabRef/jabref/pull/16838) -- We added `jabkit get-fulltext` to download and link full text PDFs, reporting the result per entry. [koppor#770](https://github.com/JabRef/jabref-koppor/pull/770) +- We added `jabkit get-fulltext` to download and link full text PDFs, reporting the result per entry. [#770](https://github.com/JabRef/jabref-koppor/pull/770) ### Changed From 04b596944e2a48ed028ec9c0e20b1b94f252f2b6 Mon Sep 17 00:00:00 2001 From: Oliver Kopp Date: Tue, 22 Sep 2026 15:04:01 +0200 Subject: [PATCH 5/5] Rename get-fulltext to get-fulltexts Matches the plural of get-cited-works and get-citing-works; the command handles all entries of a library. Co-Authored-By: Claude Opus 5 (1M context) --- .jbang/JabKitLauncher.java | 2 +- CHANGELOG.md | 2 +- docs/requirements/cli.md | 4 ++-- .../commands/{GetFulltext.java => GetFulltexts.java} | 6 +++--- .../src/main/java/org/jabref/toolkit/commands/JabKit.java | 2 +- .../{GetFulltextTest.java => GetFulltextsTest.java} | 6 +++--- .../org/jabref/logic/externalfiles/FulltextDownloader.java | 2 +- skills/users/jabkit/SKILL.md | 4 ++-- 8 files changed, 14 insertions(+), 14 deletions(-) rename jabkit/src/main/java/org/jabref/toolkit/commands/{GetFulltext.java => GetFulltexts.java} (95%) rename jabkit/src/test/java/org/jabref/toolkit/commands/{GetFulltextTest.java => GetFulltextsTest.java} (93%) diff --git a/.jbang/JabKitLauncher.java b/.jbang/JabKitLauncher.java index b825d7b8bd12..94c45558b2f3 100755 --- a/.jbang/JabKitLauncher.java +++ b/.jbang/JabKitLauncher.java @@ -32,7 +32,7 @@ //SOURCES ../jabkit/src/main/java/org/jabref/toolkit/commands/GenerateCitationKeys.java //SOURCES ../jabkit/src/main/java/org/jabref/toolkit/commands/GetCitedWorks.java //SOURCES ../jabkit/src/main/java/org/jabref/toolkit/commands/GetCitingWorks.java -//SOURCES ../jabkit/src/main/java/org/jabref/toolkit/commands/GetFulltext.java +//SOURCES ../jabkit/src/main/java/org/jabref/toolkit/commands/GetFulltexts.java //SOURCES ../jabkit/src/main/java/org/jabref/toolkit/commands/Git.java //SOURCES ../jabkit/src/main/java/org/jabref/toolkit/commands/GitMergeDriver.java //SOURCES ../jabkit/src/main/java/org/jabref/toolkit/commands/InputOption.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 8093682e7edf..ddfd30e84cf3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,7 +12,7 @@ Note that this project **does not** adhere to [Semantic Versioning](https://semv ### Added - We added `jabkit git merge-driver`, a Git merge driver that merges `.bib` files semantically. [#16838](https://github.com/JabRef/jabref/pull/16838) -- We added `jabkit get-fulltext` to download and link full text PDFs, reporting the result per entry. [#770](https://github.com/JabRef/jabref-koppor/pull/770) +- We added `jabkit get-fulltexts` to download and link full text PDFs, reporting the result per entry. [#770](https://github.com/JabRef/jabref-koppor/pull/770) ### Changed diff --git a/docs/requirements/cli.md b/docs/requirements/cli.md index 2ef90acdf62a..628ce9466335 100644 --- a/docs/requirements/cli.md +++ b/docs/requirements/cli.md @@ -94,9 +94,9 @@ are parsed correctly by the GitHub Actions runner. Needs: impl ## Full text download reports each entry -`req~jabkit.cli.get-fulltext-report~1` +`req~jabkit.cli.get-fulltexts-report~1` -`jabkit get-fulltext` updates the given library in place (or writes to `--output`). +`jabkit get-fulltexts` updates the given library in place (or writes to `--output`). It prints one line per entry saying whether its full text document was downloaded, not found, skipped, or failed, followed by a summary with the counts. Entries that already link a local PDF are skipped without contacting a publisher. diff --git a/jabkit/src/main/java/org/jabref/toolkit/commands/GetFulltext.java b/jabkit/src/main/java/org/jabref/toolkit/commands/GetFulltexts.java similarity index 95% rename from jabkit/src/main/java/org/jabref/toolkit/commands/GetFulltext.java rename to jabkit/src/main/java/org/jabref/toolkit/commands/GetFulltexts.java index 6630f0111060..02c45fa510d6 100644 --- a/jabkit/src/main/java/org/jabref/toolkit/commands/GetFulltext.java +++ b/jabkit/src/main/java/org/jabref/toolkit/commands/GetFulltexts.java @@ -22,9 +22,9 @@ import picocli.CommandLine.Option; import picocli.CommandLine.ParentCommand; -// [impl->req~jabkit.cli.get-fulltext-report~1] -@Command(name = "get-fulltext", description = "Download the full text PDFs of the entries of a library and link them.") -class GetFulltext implements Callable { +// [impl->req~jabkit.cli.get-fulltexts-report~1] +@Command(name = "get-fulltexts", description = "Download the full text PDFs of the entries of a library and link them.") +class GetFulltexts implements Callable { protected FulltextDownloader fulltextDownloader; diff --git a/jabkit/src/main/java/org/jabref/toolkit/commands/JabKit.java b/jabkit/src/main/java/org/jabref/toolkit/commands/JabKit.java index 51707ff69727..f5c54faf98a7 100644 --- a/jabkit/src/main/java/org/jabref/toolkit/commands/JabKit.java +++ b/jabkit/src/main/java/org/jabref/toolkit/commands/JabKit.java @@ -23,7 +23,7 @@ GenerateBibFromAux.class, GetCitedWorks.class, GetCitingWorks.class, - GetFulltext.class, + GetFulltexts.class, Git.class, Pdf.class, Preferences.class, diff --git a/jabkit/src/test/java/org/jabref/toolkit/commands/GetFulltextTest.java b/jabkit/src/test/java/org/jabref/toolkit/commands/GetFulltextsTest.java similarity index 93% rename from jabkit/src/test/java/org/jabref/toolkit/commands/GetFulltextTest.java rename to jabkit/src/test/java/org/jabref/toolkit/commands/GetFulltextsTest.java index b7d00a5ef0f8..3894eedd5a1d 100644 --- a/jabkit/src/test/java/org/jabref/toolkit/commands/GetFulltextTest.java +++ b/jabkit/src/test/java/org/jabref/toolkit/commands/GetFulltextsTest.java @@ -23,7 +23,7 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; -class GetFulltextTest extends AbstractJabKitTest { +class GetFulltextsTest extends AbstractJabKitTest { @TempDir private Path tempDir; @@ -32,7 +32,7 @@ class GetFulltextTest extends AbstractJabKitTest { @BeforeEach void setupDownloader() { - GetFulltext sut = new GetFulltext() { + GetFulltexts sut = new GetFulltexts() { @Override void initFields() { this.fulltextDownloader = downloader; @@ -63,7 +63,7 @@ void updatesLibraryInPlaceAndReportsEachEntry() throws IOException { return new FulltextDownloader.Result.NotFound(); }); - int exitCode = commandLine.executeToLog("get-fulltext", library.toString()); + int exitCode = commandLine.executeToLog("get-fulltexts", library.toString()); assertEquals(CommandLine.ExitCode.OK, exitCode); String output = commandLine.getStandardOutput(); diff --git a/jablib/src/main/java/org/jabref/logic/externalfiles/FulltextDownloader.java b/jablib/src/main/java/org/jabref/logic/externalfiles/FulltextDownloader.java index 2e32e0559a60..cce1a90baaae 100644 --- a/jablib/src/main/java/org/jabref/logic/externalfiles/FulltextDownloader.java +++ b/jablib/src/main/java/org/jabref/logic/externalfiles/FulltextDownloader.java @@ -26,7 +26,7 @@ /// Searches the full text PDF of an entry online, downloads it into the library's file directory and links it. /// -/// UI-independent, so that `jabkit get-fulltext` can use it; the GUI's "Search full text documents online" still has +/// UI-independent, so that `jabkit get-fulltexts` can use it; the GUI's "Search full text documents online" still has /// its own implementation. @NullMarked public class FulltextDownloader { diff --git a/skills/users/jabkit/SKILL.md b/skills/users/jabkit/SKILL.md index 7a1a171de1ae..237359644bb8 100644 --- a/skills/users/jabkit/SKILL.md +++ b/skills/users/jabkit/SKILL.md @@ -89,7 +89,7 @@ Place before the subcommand: | `generate-bib-from-aux` | Extract the subset of a library cited in a LaTeX `.aux` file | | `get-cited-works DOI` | List the works cited by a publication | | `get-citing-works DOI` | List the works citing a publication | -| `get-fulltext FILE` | Download the full text PDFs of a library's entries and link them (updates FILE in place) | +| `get-fulltexts FILE` | Download the full text PDFs of a library's entries and link them (updates FILE in place) | | `pdf extract-references FILE...` | Parse the "References" section of PDFs into BibTeX entries | | `pdf update` | Write XMP metadata and/or embedded BibTeX into linked PDFs | | `preferences reset\|import\|export` | Manage jabkit preferences | @@ -123,7 +123,7 @@ jabkit -p pdf extract-references paper.pdf jabkit pdf update --citation-key Smith2020 --input library.bib --input-format bibtex # Download and link the full text PDFs (entries with a linked PDF are skipped) -jabkit get-fulltext library.bib +jabkit get-fulltexts library.bib # Library subset actually cited in a LaTeX document jabkit generate-bib-from-aux --aux paper.aux --input full-library.bib --output paper.bib