Add jabkit pdf check-references

.jbang/JabKitLauncher.java

@@ -37,6 +37,7 @@
 //SOURCES ../jabkit/src/main/java/org/jabref/toolkit/commands/InputOption.java
 //SOURCES ../jabkit/src/main/java/org/jabref/toolkit/commands/JabKit.java
 //SOURCES ../jabkit/src/main/java/org/jabref/toolkit/commands/Pdf.java
+//SOURCES ../jabkit/src/main/java/org/jabref/toolkit/commands/PdfCheckReferences.java
 //SOURCES ../jabkit/src/main/java/org/jabref/toolkit/commands/PdfExtractReferences.java
 //SOURCES ../jabkit/src/main/java/org/jabref/toolkit/commands/PdfUpdate.java
 //SOURCES ../jabkit/src/main/java/org/jabref/toolkit/commands/Preferences.java

CHANGELOG.md

@@ -16,6 +16,7 @@ Note that this project **does not** adhere to [Semantic Versioning](https://semv
 - We added ARM64 Flatpak builds, nightly, beta, and stable channels, and host editor integration. [#13349](https://github.com/JabRef/jabref/issues/13349)
 - We added `jabkit git merge-driver`, a Git merge driver that merges `.bib` files semantically. [#16838](https://github.com/JabRef/jabref/pull/16838)
 - Extracting references from a PDF now recognizes bibliographies typeset with biblatex, including alphabetic labels such as `[AL26]`. [#775](https://github.com/JabRef/jabref-koppor/pull/775)
+- We added `jabkit pdf check-references`, which reports faults of a PDF's printed reference list, such as DOIs given as URL. [#778](https://github.com/JabRef/jabref-koppor/pull/778)
 - We added tooltips with action descriptions to menu items. [#9615](https://github.com/JabRef/jabref/issues/9615)
 - We added "Contribute to JabRef" to the Help menu, linking to [contribute.jabref.org](https://contribute.jabref.org). [#17162](https://github.com/JabRef/jabref/pull/17162)
 - We added EasyOCR, PaddleOCR, and AppleOCR as selectable OCR engines, alongside Tesseract and Docling. [#16866](https://github.com/JabRef/jabref/issues/16866)

docs/requirements/cli.md

@@ -92,4 +92,15 @@ are parsed correctly by the GitHub Actions runner.
 
 Needs: impl
 
+## Faults of the reference list printed in a PDF
+`req~jabkit.cli.pdf-check-references~1`
+
+`jabkit pdf check-references FILE` reads the reference list printed in a PDF and reports faults of the underlying bibliographic data, one line per finding, starting with the reference's printed label:
+a DOI given as URL, a missing venue that hints at a wrong entry type, a volume inside a journal name, a URL of an index such as Semantic Scholar instead of the publisher, and fields printed for some references of an entry type but not for the others.
+Unless `--no-online` is given, it additionally reports DOIs that Crossref knows for references printed without one, and words whose capitals were lost, such as "Iot" instead of "IoT".
+
+The command exits with 1 if there are findings, so that CI can fail on them.
+
+Needs: impl, utest
+
 <!-- markdownlint-disable-file MD022 -->

jabkit/src/main/java/org/jabref/toolkit/commands/Pdf.java

@@ -11,6 +11,7 @@
 @Command(name = "pdf", description = "Manage PDF metadata.",
         subcommands = {
                 PdfUpdate.class,
+                PdfCheckReferences.class,
                 PdfExtractReferences.class
                 // RemoveComments.class
                 // RemoveEmbedded.class
@@ -25,7 +26,7 @@ class Pdf implements Callable<Integer> {
 
     @Override
     public Integer call() {
-        System.err.println(Localization.lang("Specify a subcommand (update, extract-references)."));
+        System.err.println(Localization.lang("Specify a subcommand (update, check-references, extract-references)."));
         return 2;
     }
 }

jabkit/src/main/java/org/jabref/toolkit/commands/PdfCheckReferences.java

@@ -0,0 +1,68 @@
+package org.jabref.toolkit.commands;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.List;
+import java.util.concurrent.Callable;
+
+import org.jabref.logic.importer.fetcher.CrossRef;
+import org.jabref.logic.l10n.Localization;
+import org.jabref.logic.quality.consistency.PrintedReferencesCheck;
+import org.jabref.toolkit.converter.CygWinPathConverter;
+
+import org.jspecify.annotations.NullMarked;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import picocli.CommandLine;
+
+import static picocli.CommandLine.Command;
+import static picocli.CommandLine.Mixin;
+import static picocli.CommandLine.Option;
+import static picocli.CommandLine.Parameters;
+import static picocli.CommandLine.ParentCommand;
+
+// [impl->req~jabkit.cli.pdf-check-references~1]
+@NullMarked
+@Command(name = "check-references", description = "Report faults in the reference list printed in a PDF, such as DOIs given as URL, missing DOIs, and fields printed for some references only.")
+class PdfCheckReferences implements Callable<Integer> {
+
+    private static final Logger LOGGER = LoggerFactory.getLogger(PdfCheckReferences.class);
+
+    @ParentCommand
+    protected Pdf pdf;
+
+    @Mixin
+    private JabKit.SharedOptions sharedOptions;
+
+    @Parameters(paramLabel = "FILE", converter = CygWinPathConverter.class, description = "PDF whose reference list is checked.")
+    private Path inputFile;
+
+    @Option(names = "--online", negatable = true, defaultValue = "true", fallbackValue = "true",
+            description = "Look up missing DOIs and title capitalization at Crossref (default: ${DEFAULT-VALUE}).")
+    private boolean online;
+
+    @Override
+    public Integer call() {
+        if (!Files.exists(inputFile)) {
+            System.err.println(Localization.lang("File %0 not found.", inputFile.toString()));
+            return CommandLine.ExitCode.USAGE;
+        }
+
+        PrintedReferencesCheck check = new PrintedReferencesCheck(online ? new CrossRef(pdf.argumentProcessor.cliPreferences.getImporterPreferences()) : null);
+        List<PrintedReferencesCheck.Finding> findings;
+        try {
+            findings = check.check(inputFile);
+        } catch (IOException e) {
+            LOGGER.error("Could not read {}", inputFile, e);
+            return CommandLine.ExitCode.SOFTWARE;
+        }
+
+        findings.forEach(finding -> System.out.println(finding.reference() + " " + finding.message()));
+        if (!sharedOptions.porcelain) {
+            System.out.println(Localization.lang("%0 finding(s).", findings.size()));
+        }
+        // Findings give a non-zero exit, so CI can fail on them - same as "jabkit check"
+        return findings.isEmpty() ? CommandLine.ExitCode.OK : 1;
+    }
+}

jablib/src/main/java/org/jabref/logic/quality/consistency/PrintedReferencesCheck.java

@@ -0,0 +1,162 @@
+package org.jabref.logic.quality.consistency;
+
+import java.io.IOException;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Optional;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+
+import org.jabref.logic.importer.FetcherException;
+import org.jabref.logic.importer.fetcher.CrossRef;
+import org.jabref.logic.importer.fileformat.pdf.RuleBasedBibliographyPdfImporter;
+import org.jabref.logic.l10n.Localization;
+import org.jabref.model.database.BibDatabaseContext;
+import org.jabref.model.entry.BibEntry;
+import org.jabref.model.entry.BibEntryTypesManager;
+import org.jabref.model.entry.field.Field;
+import org.jabref.model.entry.field.StandardField;
+import org.jabref.model.entry.identifier.DOI;
+import org.jabref.model.entry.types.EntryType;
+import org.jabref.model.entry.types.StandardEntryType;
+
+import org.jspecify.annotations.NullMarked;
+import org.jspecify.annotations.Nullable;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/// Checks the reference list printed in a PDF, e.g., the bibliography of a paper under review.
+///
+/// The references are read by [RuleBasedBibliographyPdfImporter], which keeps field values as printed.
+/// Thus, faults of the underlying BibTeX data remain visible, such as a DOI given as URL.
+///
+/// [impl->req~jabkit.cli.pdf-check-references~1]
+@NullMarked
+public class PrintedReferencesCheck {
+
+    /// @param reference the printed label of the reference (`[AL26]`), or the entry type for findings about several references (`@article`)
+    public record Finding(String reference, String message) {
+    }
+
+    private static final Logger LOGGER = LoggerFactory.getLogger(PrintedReferencesCheck.class);
+
+    private static final Pattern VOLUME_IN_NAME = Pattern.compile("\\bVolume \\d+");
+    // Search engines and social networks list publications, but do not publish them
+    private static final Pattern INDEX_URL = Pattern.compile("^https?://(?:[^/]*\\.)?(academia\\.edu|dblp\\.org|researchgate\\.net|scholar\\.google\\.[a-z.]+|semanticscholar\\.org)(?:[/:?]|$)");
+    private static final Pattern INNER_UPPERCASE = Pattern.compile(".\\p{Lu}");
+
+    private final @Nullable CrossRef crossRef;
+
+    /// @param crossRef used to find missing DOIs and to compare the capitalization of titles; `null` to check offline
+    public PrintedReferencesCheck(@Nullable CrossRef crossRef) {
+        this.crossRef = crossRef;
+    }
+
+    public List<Finding> check(Path pdf) throws IOException {
+        return check(new RuleBasedBibliographyPdfImporter().importDatabase(pdf).getDatabaseContext());
+    }
+
+    /// @param references entries as returned by [RuleBasedBibliographyPdfImporter] without citation key generation: the citation key is the printed label
+    public List<Finding> check(BibDatabaseContext references) {
+        List<Finding> findings = new ArrayList<>();
+        for (BibEntry entry : references.getEntries()) {
+            String label = label(entry);
+            entry.getField(StandardField.DOI)
+                 .filter(doi -> doi.startsWith("http"))
+                 .ifPresent(doi -> findings.add(new Finding(label, Localization.lang("DOI is given as URL '%0'. Put only the DOI into the field, the style adds the link.", doi))));
+            if ((entry.getType() == StandardEntryType.Article && !entry.hasField(StandardField.JOURNAL))
+                    || (entry.getType() == StandardEntryType.InProceedings && !entry.hasField(StandardField.BOOKTITLE))) {
+                findings.add(new Finding(label, Localization.lang("Nothing is printed after 'In:'. The venue is missing, or the entry type is wrong: use 'online' or 'misc' for web resources and preprints.")));
+            }
+            entry.getField(StandardField.URL)
+                 .map(INDEX_URL::matcher)
+                 .filter(Matcher::find)
+                 .ifPresent(indexUrl -> findings.add(new Finding(label, Localization.lang("URL points to %0, which indexes publications, but does not publish them. Link the publisher's page or give the DOI.", indexUrl.group(1)))));
+            entry.getField(StandardField.JOURNAL)
+                 .filter(journal -> VOLUME_IN_NAME.matcher(journal).find())
+                 .ifPresent(journal -> findings.add(new Finding(label, Localization.lang("Journal name '%0' contains the volume. Put the volume into the volume field.", journal))));
+            if (crossRef != null) {
+                findings.addAll(checkAgainstCrossRef(crossRef, label, entry));
+            }
+        }
+        findings.addAll(checkConsistency(references));
+        return findings;
+    }
+
+    private static List<Finding> checkAgainstCrossRef(CrossRef crossRef, String label, BibEntry entry) {
+        List<Finding> findings = new ArrayList<>();
+        try {
+            Optional<DOI> doi = entry.getField(StandardField.DOI).flatMap(DOI::parse);
+            if (doi.isEmpty()) {
+                doi = crossRef.findIdentifier(entry);
+                doi.ifPresent(found -> findings.add(new Finding(label, Localization.lang("No DOI is printed. Crossref has DOI %0.", found.asString()))));
+            }
+            if (doi.isEmpty()) {
+                return findings;
+            }
+            Optional<String> printedTitle = entry.getField(StandardField.TITLE);
+            Optional<String> registeredTitle = crossRef.performSearchById(doi.get().asString()).flatMap(registered -> registered.getField(StandardField.TITLE));
+            if (printedTitle.isPresent() && registeredTitle.isPresent()) {
+                String lostCapitals = lostCapitals(printedTitle.get(), registeredTitle.get());
+                if (!lostCapitals.isEmpty()) {
+                    findings.add(new Finding(label, Localization.lang("Capitalization differs from Crossref: %0. Protect these words with braces in the title field.", lostCapitals)));
+                }
+            }
+        } catch (FetcherException e) {
+            LOGGER.warn("Could not look up {} at Crossref", label, e);
+        }
+        return findings;
+    }
+
+    /// Words the printed title writes in a different case than the registered one, restricted to words with a capital letter
+    /// after the first one ("IoT", "GitHub"). Other case differences are the choice of the bibliography style (title case, sentence case).
+    ///
+    /// @return `'Iot' should be 'IoT'`, several joined by `, `; empty if there are none
+    static String lostCapitals(String printedTitle, String registeredTitle) {
+        String[] printedWords = printedTitle.split("\\s+");
+        String[] registeredWords = registeredTitle.split("\\s+");
+        if (printedWords.length != registeredWords.length) {
+            return "";
+        }
+        List<String> differences = new ArrayList<>();
+        for (int i = 0; i < printedWords.length; i++) {
+            String printed = printedWords[i];
+            String registered = registeredWords[i];
+            if (printed.equalsIgnoreCase(registered) && !printed.equals(registered) && INNER_UPPERCASE.matcher(registered).find()) {
+                differences.add(Localization.lang("'%0' should be '%1'", printed, registered));
+            }
+        }
+        return String.join(", ", differences);
+    }
+
+    /// References of the same type should list the same fields, e.g., all articles an ISSN or none.
+    private static List<Finding> checkConsistency(BibDatabaseContext references) {
+        BibliographyConsistencyCheck.Result result = new BibliographyConsistencyCheck().check(references, new BibEntryTypesManager(), (_, _) -> {
+        });
+        List<Finding> findings = new ArrayList<>();
+        // The result map is filled from a HashMap; sorting keeps the output stable
+        result.entryTypeToResultMap().entrySet().stream().sorted(Comparator.comparing(typeResult -> typeResult.getKey().getName())).forEach(typeResult -> {
+            EntryType entryType = typeResult.getKey();
+            BibliographyConsistencyCheck.EntryTypeResult entryTypeResult = typeResult.getValue();
+            List<BibEntry> entriesOfType = references.getEntries().stream().filter(entry -> entry.getType().equals(entryType)).toList();
+            entryTypeResult.fields().stream().sorted(Comparator.comparing(Field::getName)).forEach(field ->
+                    findings.add(new Finding("@" + entryType.getName(), Localization.lang("'%0' is printed for %1, but not for %2.",
+                            field.getName(), labels(entriesOfType, field, true), labels(entriesOfType, field, false)))));
+        });
+        return findings;
+    }
+
+    private static String labels(List<BibEntry> entries, Field field, boolean withField) {
+        return entries.stream()
+                      .filter(entry -> entry.hasField(field) == withField)
+                      .map(PrintedReferencesCheck::label)
+                      .collect(Collectors.joining(", "));
+    }
+
+    private static String label(BibEntry entry) {
+        return "[" + entry.getCitationKey().orElse("?") + "]";
+    }
+}

jablib/src/main/resources/l10n/JabRef_en.properties

@@ -3713,7 +3713,7 @@ Click\ a\ cell\ to\ edit\ the\ native\ query=Click a cell to edit the native que
 
 Specify\ a\ subcommand\ (consistency,\ integrity)\ or\ an\ input\ file.=Specify a subcommand (consistency, integrity) or an input file.
 Specify\ a\ subcommand\ (reset,\ import,\ export).=Specify a subcommand (reset, import, export).
-Specify\ a\ subcommand\ (update,\ extract-references).=Specify a subcommand (update, extract-references).
+Specify\ a\ subcommand\ (update,\ check-references,\ extract-references).=Specify a subcommand (update, check-references, extract-references).
 The\ format\ option\ must\ contain\ either\ 'xmp'\ or\ 'bibtex-attachment'.=The format option must contain either 'xmp' or 'bibtex-attachment'.
 Importer\ for\ the\ Hayagriva\ YAML\ format.=Importer for the Hayagriva YAML format.
 Normalize\ keyword\ delimiters=Normalize keyword delimiters
@@ -3741,3 +3741,12 @@ Move\ group=Move group
 Sort\ subgroups=Sort subgroups
 Change\ string\ constants=Change string constants
 Founded\ by\ Morten\ O.\ Alver\ and\ Nizar\ Batada.=Founded by Morten O. Alver and Nizar Batada.
+DOI\ is\ given\ as\ URL\ '%0'.\ Put\ only\ the\ DOI\ into\ the\ field,\ the\ style\ adds\ the\ link.=DOI is given as URL '%0'. Put only the DOI into the field, the style adds the link.
+Nothing\ is\ printed\ after\ 'In\:'.\ The\ venue\ is\ missing,\ or\ the\ entry\ type\ is\ wrong\:\ use\ 'online'\ or\ 'misc'\ for\ web\ resources\ and\ preprints.=Nothing is printed after 'In:'. The venue is missing, or the entry type is wrong: use 'online' or 'misc' for web resources and preprints.
+URL\ points\ to\ %0,\ which\ indexes\ publications,\ but\ does\ not\ publish\ them.\ Link\ the\ publisher's\ page\ or\ give\ the\ DOI.=URL points to %0, which indexes publications, but does not publish them. Link the publisher's page or give the DOI.
+Journal\ name\ '%0'\ contains\ the\ volume.\ Put\ the\ volume\ into\ the\ volume\ field.=Journal name '%0' contains the volume. Put the volume into the volume field.
+No\ DOI\ is\ printed.\ Crossref\ has\ DOI\ %0.=No DOI is printed. Crossref has DOI %0.
+Capitalization\ differs\ from\ Crossref\:\ %0.\ Protect\ these\ words\ with\ braces\ in\ the\ title\ field.=Capitalization differs from Crossref: %0. Protect these words with braces in the title field.
+'%0'\ should\ be\ '%1'='%0' should be '%1'
+'%0'\ is\ printed\ for\ %1,\ but\ not\ for\ %2.='%0' is printed for %1, but not for %2.
+%0\ finding(s).=%0 finding(s).