Convert .bib libraries into directory libraries

docs/requirements/directory-library.md

@@ -94,7 +94,7 @@ the renamed files.
 Needs: impl
 
 ## The library is mirrored into a single .bib file
-`req~directory-library.bib-mirror~1`
+`req~directory-library.bib-mirror~2`
 
 A directory library is continuously mirrored into `<root>/<root-name>.bib` (debounced with the
 sidecar write-back), so plain BibTeX consumers and collaborators can read and edit the library
@@ -107,6 +107,25 @@ cancelled resolution keeps the library's state. A pre-existing `.bib` without a
 adopted against an empty base, which can only add entries or raise conflicts, never delete
 library content. Entries are matched across the mirror by citation key; entries without one are
 not matched. The mirror itself is recreated when deleted and never imported as a sidecar.
+User-defined groups are carried by the mirror's metadata block: they are written with every
+mirror update and restored into the library when it is opened (the automatic directory-structure
+group is not duplicated).
+
+Needs: impl, utest
+
+## A .bib library converts into a directory library
+`req~directory-library.convert~1`
+
+A saved local `.bib` library can be converted into a directory library. The root is the
+library-specific file directory when configured, otherwise the `.bib` file's directory. The
+conversion only proceeds when every linked local file resolves to a location under that root and
+the library carries no BibTeX strings or preamble; otherwise the obstacles are reported and
+nothing is changed. On conversion, every entry gets a single-entry Markdown sidecar next to its
+first linked file (or named after its citation key in the root), the `.bib` file moves to the
+root as `<root-name>.bib` and becomes the library's mirror (preserving groups via the mirror
+metadata) together with its merge base, and the root is reopened as a directory library without
+a merge. Sidecars pair with a PDF of the same base name only: further linked files, and a first
+linked file that is no PDF, survive solely through the mirror.
 
 Needs: impl, utest
 

jabgui/src/main/java/org/jabref/gui/actions/StandardActions.java

@@ -81,6 +81,7 @@ public enum StandardActions implements Action {
     NEW_LIBRARY(Localization.lang("New empty library"), IconTheme.JabRefIcons.NEW),
     OPEN_LIBRARY(Localization.lang("Open library..."), IconTheme.JabRefIcons.OPEN, KeyBinding.OPEN_LIBRARY),
     OPEN_FOLDER_AS_LIBRARY(Localization.lang("Open folder as library..."), IconTheme.JabRefIcons.OPEN),
+    CONVERT_TO_FOLDER_LIBRARY(Localization.lang("Convert to folder library..."), IconTheme.JabRefIcons.FOLDER),
     MERGE_LIBRARY(Localization.lang("Merge..."), IconTheme.JabRefIcons.MERGE_ENTRIES),
     IMPORT(Localization.lang("Import"), IconTheme.JabRefIcons.IMPORT),
     EXPORT(Localization.lang("Export"), IconTheme.JabRefIcons.EXPORT, KeyBinding.EXPORT),

jabgui/src/main/java/org/jabref/gui/frame/MainMenu.java

@@ -48,6 +48,7 @@
 import org.jabref.gui.help.SearchForUpdateAction;
 import org.jabref.gui.importer.NewDatabaseAction;
 import org.jabref.gui.importer.NewEntryAction;
+import org.jabref.gui.importer.actions.ConvertToDirectoryLibraryAction;
 import org.jabref.gui.importer.actions.ImportCommand;
 import org.jabref.gui.importer.actions.OpenDatabaseAction;
 import org.jabref.gui.importer.actions.OpenDirectoryLibraryAction;
@@ -166,14 +167,16 @@ private void createMenu() {
         Menu tools = new Menu(Localization.lang("Tools"));
         Menu help = new Menu(Localization.lang("Help"));
 
+        OpenDirectoryLibraryAction openDirectoryLibraryAction = new OpenDirectoryLibraryAction(frame, dialogService, preferences, aiService, stateManager, fileUpdateMonitor, entryTypesManager, gitHandlerRegistry, clipBoardManager, taskExecutor);
         file.getItems().addAll(
                 factory.createMenuItem(StandardActions.NEW_LIBRARY, new NewDatabaseAction(frame, preferences)),
                 factory.createMenuItem(StandardActions.OPEN_LIBRARY, openDatabaseActionSupplier.get()),
-                factory.createMenuItem(StandardActions.OPEN_FOLDER_AS_LIBRARY, new OpenDirectoryLibraryAction(frame, dialogService, preferences, aiService, stateManager, fileUpdateMonitor, entryTypesManager, gitHandlerRegistry, clipBoardManager, taskExecutor)),
+                factory.createMenuItem(StandardActions.OPEN_FOLDER_AS_LIBRARY, openDirectoryLibraryAction),
                 fileHistoryMenu,
                 factory.createMenuItem(StandardActions.SAVE_LIBRARY, new SaveAction(SaveAction.SaveMethod.SAVE, frame::getCurrentLibraryTab, dialogService, preferences, stateManager, entryTypesManager, journalAbbreviationRepository)),
                 factory.createMenuItem(StandardActions.SAVE_LIBRARY_AS, new SaveAction(SaveAction.SaveMethod.SAVE_AS, frame::getCurrentLibraryTab, dialogService, preferences, stateManager, entryTypesManager, journalAbbreviationRepository)),
                 factory.createMenuItem(StandardActions.SAVE_ALL, new SaveAllAction(frame::getLibraryTabs, preferences, dialogService, stateManager, entryTypesManager, journalAbbreviationRepository)),
+                factory.createMenuItem(StandardActions.CONVERT_TO_FOLDER_LIBRARY, new ConvertToDirectoryLibraryAction(frame, dialogService, preferences, stateManager, entryTypesManager, journalAbbreviationRepository, openDirectoryLibraryAction, taskExecutor)),
                 factory.createMenuItem(StandardActions.CLOSE_LIBRARY, new JabRefFrame.CloseDatabaseAction(frame, stateManager)),
 
                 new SeparatorMenuItem(),

jabgui/src/main/java/org/jabref/gui/importer/actions/ConvertToDirectoryLibraryAction.java

@@ -0,0 +1,125 @@
+package org.jabref.gui.importer.actions;
+
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.List;
+import java.util.stream.Collectors;
+
+import org.jabref.gui.DialogService;
+import org.jabref.gui.LibraryTab;
+import org.jabref.gui.LibraryTabContainer;
+import org.jabref.gui.StateManager;
+import org.jabref.gui.actions.ActionHelper;
+import org.jabref.gui.actions.SimpleCommand;
+import org.jabref.gui.exporter.SaveDatabaseAction;
+import org.jabref.gui.preferences.GuiPreferences;
+import org.jabref.logic.directorylibrary.DirectoryLibraryConverter;
+import org.jabref.logic.directorylibrary.DirectoryLibrarySynchronizer;
+import org.jabref.logic.journals.JournalAbbreviationRepository;
+import org.jabref.logic.l10n.Localization;
+import org.jabref.logic.util.BackgroundTask;
+import org.jabref.logic.util.TaskExecutor;
+import org.jabref.model.database.BibDatabaseContext;
+import org.jabref.model.entry.BibEntryTypesManager;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static java.util.function.Predicate.not;
+
+/// Converts the current `.bib` library into a directory library (see
+/// [DirectoryLibraryConverter]) and reopens its root as such. Only offered for saved local
+/// libraries; aborts with an explanation when the library does not fit under one root.
+public class ConvertToDirectoryLibraryAction extends SimpleCommand {
+
+    private static final Logger LOGGER = LoggerFactory.getLogger(ConvertToDirectoryLibraryAction.class);
+    private static final int MAX_REPORTED_OBSTACLES = 10;
+
+    private final LibraryTabContainer tabContainer;
+    private final DialogService dialogService;
+    private final GuiPreferences preferences;
+    private final StateManager stateManager;
+    private final BibEntryTypesManager entryTypesManager;
+    private final JournalAbbreviationRepository journalAbbreviationRepository;
+    private final OpenDirectoryLibraryAction openDirectoryLibraryAction;
+    private final TaskExecutor taskExecutor;
+    private final DirectoryLibraryConverter converter = new DirectoryLibraryConverter();
+
+    public ConvertToDirectoryLibraryAction(LibraryTabContainer tabContainer,
+                                           DialogService dialogService,
+                                           GuiPreferences preferences,
+                                           StateManager stateManager,
+                                           BibEntryTypesManager entryTypesManager,
+                                           JournalAbbreviationRepository journalAbbreviationRepository,
+                                           OpenDirectoryLibraryAction openDirectoryLibraryAction,
+                                           TaskExecutor taskExecutor) {
+        this.tabContainer = tabContainer;
+        this.dialogService = dialogService;
+        this.preferences = preferences;
+        this.stateManager = stateManager;
+        this.entryTypesManager = entryTypesManager;
+        this.journalAbbreviationRepository = journalAbbreviationRepository;
+        this.openDirectoryLibraryAction = openDirectoryLibraryAction;
+        this.taskExecutor = taskExecutor;
+
+        this.executable.bind(ActionHelper.needsSavedLocalDatabase(stateManager));
+    }
+
+    @Override
+    public void execute() {
+        LibraryTab libraryTab = tabContainer.getCurrentLibraryTab();
+        BibDatabaseContext context = libraryTab.getBibDatabaseContext();
+        DirectoryLibraryConverter.determineRoot(context, preferences.getFilePreferences())
+                                 .ifPresent(root -> convert(libraryTab, context, root));
+    }
+
+    private void convert(LibraryTab libraryTab, BibDatabaseContext context, Path root) {
+        String title = Localization.lang("Convert to folder library");
+        List<String> obstacles = converter.obstacles(context, root, preferences.getFilePreferences());
+        if (!obstacles.isEmpty()) {
+            dialogService.showErrorDialogAndWait(title,
+                    Localization.lang("The library cannot be converted.") + "\n\n" + reportedObstacles(obstacles));
+            return;
+        }
+
+        Path mirrorTarget = root.resolve(DirectoryLibrarySynchronizer.mirrorFileName(root));
+        boolean overwritesForeignFile = context.getDatabasePath().filter(not(mirrorTarget::equals)).isPresent() && Files.exists(mirrorTarget);
+        if (overwritesForeignFile) {
+            dialogService.showErrorDialogAndWait(title,
+                    Localization.lang("'%0' already exists and would be overwritten.", mirrorTarget.toString()));
+            return;
+        }
+
+        boolean confirmed = dialogService.showConfirmationDialogAndWait(title,
+                Localization.lang("Every entry gets a Markdown sidecar next to its linked file, and the library file moves to '%0', staying in sync with the folder from now on.", mirrorTarget.toString()));
+        if (!confirmed) {
+            return;
+        }
+        if (new SaveDatabaseAction(libraryTab, dialogService, preferences, entryTypesManager, stateManager, journalAbbreviationRepository).save() != SaveDatabaseAction.SaveResult.SUCCESS) {
+            return;
+        }
+
+        BackgroundTask.wrap(() -> converter.convert(context, root, preferences.getFilePreferences()))
+                      .onSuccess(_ -> {
+                          if (!tabContainer.closeTab(libraryTab)) {
+                              LOGGER.warn("The converted library's tab stays open although its file moved to {}", mirrorTarget);
+                          }
+                          openDirectoryLibraryAction.openDirectory(root);
+                      })
+                      .onFailure(exception -> {
+                          LOGGER.error("Could not convert the library to a folder library at {}", root, exception);
+                          dialogService.showErrorDialogAndWait(title, exception);
+                      })
+                      .executeWith(taskExecutor);
+    }
+
+    private static String reportedObstacles(List<String> obstacles) {
+        String reported = obstacles.stream()
+                                   .limit(MAX_REPORTED_OBSTACLES)
+                                   .collect(Collectors.joining("\n"));
+        if (obstacles.size() > MAX_REPORTED_OBSTACLES) {
+            return reported + "\n" + Localization.lang("... and %0 more", Integer.toString(obstacles.size() - MAX_REPORTED_OBSTACLES));
+        }
+        return reported;
+    }
+}

jablib/src/main/java/org/jabref/logic/directorylibrary/DirectoryLibraryConverter.java

@@ -0,0 +1,139 @@
+package org.jabref.logic.directorylibrary;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.StandardCopyOption;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Optional;
+
+import org.jabref.logic.FilePreferences;
+import org.jabref.logic.exporter.HayagrivaEntryWriter;
+import org.jabref.logic.l10n.Localization;
+import org.jabref.logic.util.io.FileNameCleaner;
+import org.jabref.logic.util.io.FileUtil;
+import org.jabref.model.database.BibDatabaseContext;
+import org.jabref.model.entry.BibEntry;
+import org.jabref.model.entry.LinkedFile;
+
+import org.jspecify.annotations.NullMarked;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static java.util.function.Predicate.not;
+
+/// Converts a regular `.bib` library into a directory library: every entry gets a Markdown
+/// sidecar (see [MarkdownSidecar]) next to its linked file, and the `.bib` itself becomes the
+/// library's mirror. The conversion is only offered when the whole library fits under one
+/// root — [#obstacles] lists everything that prevents it.
+// [impl->req~directory-library.convert~1]
+@NullMarked
+public class DirectoryLibraryConverter {
+
+    private static final Logger LOGGER = LoggerFactory.getLogger(DirectoryLibraryConverter.class);
+
+    private final MarkdownSidecar markdownSidecar = new MarkdownSidecar();
+
+    /// The directory that becomes the library root: the library-specific file directory when
+    /// one is configured, otherwise the `.bib` file's directory.
+    public static Optional<Path> determineRoot(BibDatabaseContext context, FilePreferences filePreferences) {
+        return context.getAllFileDirectories(filePreferences)
+                      .getLibraryDirectoryOpt()
+                      .or(context::getDatabaseDirectory)
+                      .map(Path::normalize);
+    }
+
+    /// Everything that prevents the conversion: linked files that cannot be found or do not
+    /// live under the root, and library content sidecars cannot represent (BibTeX strings,
+    /// preamble). An empty result means the library converts losslessly file-wise.
+    public List<String> obstacles(BibDatabaseContext context, Path root, FilePreferences filePreferences) {
+        List<String> obstacles = new ArrayList<>();
+        if (context.getDatabase().getPreamble().isPresent()) {
+            obstacles.add(Localization.lang("The library contains a preamble, which a folder library cannot represent."));
+        }
+        if (!context.getDatabase().getStringValues().isEmpty()) {
+            obstacles.add(Localization.lang("The library contains BibTeX strings, which a folder library cannot represent."));
+        }
+        List<Path> fileDirectories = context.getFileDirectories(filePreferences);
+        Path normalizedRoot = root.toAbsolutePath().normalize();
+        for (BibEntry entry : context.getDatabase().getEntries()) {
+            String label = entry.getCitationKey().orElseGet(() -> entry.getAuthorTitleYear(40));
+            for (LinkedFile linkedFile : entry.getFiles()) {
+                if (linkedFile.isOnlineLink()) {
+                    continue;
+                }
+                linkedFile.findIn(fileDirectories).ifPresentOrElse(resolved -> {
+                    if (!resolved.toAbsolutePath().normalize().startsWith(normalizedRoot)) {
+                        obstacles.add(Localization.lang("Linked file '%0' of entry '%1' is outside of '%2'.", linkedFile.getLink(), label, root.toString()));
+                    }
+                }, () -> obstacles.add(Localization.lang("Linked file '%0' of entry '%1' was not found.", linkedFile.getLink(), label)));
+            }
+        }
+        return obstacles;
+    }
+
+    /// Runs once the library is saved: sidecars first, then the `.bib` moves into the root as
+    /// the mirror, and a copy becomes the merge base — so reopening the root finds mirror and
+    /// base identical instead of merging the library against itself.
+    ///
+    /// @return the mirror file
+    public Path convert(BibDatabaseContext context, Path root, FilePreferences filePreferences) throws IOException {
+        Path bibFile = context.getDatabasePath().orElseThrow();
+        Path mirror = root.resolve(DirectoryLibrarySynchronizer.mirrorFileName(root));
+        writeSidecars(context, root, filePreferences);
+        if (!mirror.equals(bibFile)) {
+            Files.move(bibFile, mirror);
+        }
+        Path base = DirectoryLibrarySynchronizer.mirrorBaseFile(root);
+        Files.createDirectories(base.getParent());
+        Files.copy(mirror, base, StandardCopyOption.REPLACE_EXISTING);
+        return mirror;
+    }
+
+    /// Writes one single-entry Markdown sidecar per entry: next to the entry's first linked
+    /// file (sharing its base name, per the pairing convention), or named after the citation
+    /// key in the root. Occupied names are uniquified with a numeric suffix. A failure removes
+    /// the sidecars written so far, so a retry does not produce duplicates.
+    public void writeSidecars(BibDatabaseContext context, Path root, FilePreferences filePreferences) throws IOException {
+        List<Path> fileDirectories = context.getFileDirectories(filePreferences);
+        List<Path> written = new ArrayList<>();
+        try {
+            for (BibEntry entry : context.getDatabase().getEntries()) {
+                Path sidecar = sidecarFor(entry, root, fileDirectories);
+                String key = entry.getCitationKey().filter(not(String::isBlank)).orElse("entry");
+                String document = markdownSidecar.merge("", List.of(new HayagrivaEntryWriter.KeyedEntry("", key, entry)));
+                Files.writeString(sidecar, document);
+                written.add(sidecar);
+            }
+        } catch (IOException e) {
+            written.forEach(DirectoryLibraryConverter::deleteQuietly);
+            throw e;
+        }
+    }
+
+    private static void deleteQuietly(Path file) {
+        try {
+            Files.deleteIfExists(file);
+        } catch (IOException e) {
+            LOGGER.warn("Could not remove partially written sidecar {}", file, e);
+        }
+    }
+
+    private static Path sidecarFor(BibEntry entry, Path root, List<Path> fileDirectories) {
+        Path baseNamePath = entry.getFiles().stream()
+                                 .filter(linkedFile -> !linkedFile.isOnlineLink())
+                                 .findFirst()
+                                 .flatMap(linkedFile -> linkedFile.findIn(fileDirectories))
+                                 .map(paired -> paired.resolveSibling(FileUtil.getBaseName(paired)))
+                                 .orElseGet(() -> root.resolve(entry.getCitationKey()
+                                                                    .map(FileNameCleaner::cleanFileName)
+                                                                    .filter(not(String::isBlank))
+                                                                    .orElse("entry")));
+        Path sidecar = baseNamePath.resolveSibling(baseNamePath.getFileName() + "." + MarkdownSidecar.MARKDOWN_EXTENSION);
+        for (int counter = 1; Files.exists(sidecar); counter++) {
+            sidecar = baseNamePath.resolveSibling(baseNamePath.getFileName() + "-" + counter + "." + MarkdownSidecar.MARKDOWN_EXTENSION);
+        }
+        return sidecar;
+    }
+}