Registry and Plugin (Backend)

.github/workflows/push-continuous-integration.yml

@@ -10,14 +10,26 @@ on:
       integration_test_ref:
         description: 'Integration testing repository git commit hash or branch'
         required: true
-        default: 'dfdcd948cbc6d6bbd2db6620d8499bc8698843e3'
+        default: 'dev-registry/main'
 
 env:
   INTEGRATION_TEST_REPOSITORY: ${{ github.event.inputs.integration_test_repository || 'psilabs-dev/aio-lanraragi' }}
-  INTEGRATION_TEST_REF: ${{ github.event.inputs.integration_test_ref || 'dfdcd948cbc6d6bbd2db6620d8499bc8698843e3' }}
+  INTEGRATION_TEST_REF: ${{ github.event.inputs.integration_test_ref || 'dev-registry/main' }}
 
 name: "Continuous Integration \U0001F44C\U0001F440"
 jobs:
+  eslint:
+    name: ESLint
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v6
+    - uses: actions/setup-node@v6
+      with:
+        node-version: "22"
+        cache: npm
+    - run: npm ci
+    - run: npm run lint
+
   testSuite:
     name: Run Test Suite and Perl Critic
     runs-on: ubuntu-latest
@@ -85,18 +97,29 @@ jobs:
         run: |
           mkdir -p "$GITHUB_WORKSPACE/staging"
       - name: Run integration tests against local LANraragi build
-        timeout-minutes: 30
+        timeout-minutes: 70
         run: |
           cd aio-lanraragi/integration_tests
           export CI=true
           uv run pytest tests --log-cli-level=INFO \
             --image lanraragi:ci \
             --staging "$GITHUB_WORKSPACE/staging" \
+            --server-logs "$GITHUB_WORKSPACE/server-logs" \
+            --cache-backend valkey \
             --playwright \
+            --no-rate-limit \
+            --dev registry \
             --npseed 42 \
-            -k "not test_double_page_navigation and not test_handler_resource_management"
+            -k "not test_double_page_navigation and not test_handler_resource_management and not test_plugin_uninstall_ui and not test_sideloaded_script_lifecycle and not test_search_functionality and not test_ui_nofunmode_login_right_password and not test_ui_enable_nofunmode and not test_validation_carousel_search"
         env:
           DOCKER_HOST: unix:///var/run/docker.sock
+      - name: Upload server logs
+        if: failure()
+        uses: actions/upload-artifact@v4
+        with:
+          name: server-logs-ubuntu
+          path: ${{ github.workspace }}/server-logs/
+          retention-days: 14
 
   integrationTestsWindows:
     name: Run Windows Integration Tests
@@ -215,13 +238,23 @@ jobs:
           New-Item -ItemType Directory -Path "$env:GITHUB_WORKSPACE\staging" -Force | Out-Null
       - name: Run integration tests against local LANraragi build
         working-directory: aio-lanraragi/integration_tests
-        timeout-minutes: 50
+        timeout-minutes: 100
         run: |
           $env:CI='true'
           uv run pytest tests `
           --log-cli-level=INFO `
           --windist "$env:GITHUB_WORKSPACE\LANraragi\win-dist" `
           --staging "$env:GITHUB_WORKSPACE\staging" `
+          --server-logs "$env:GITHUB_WORKSPACE\server-logs" `
           --playwright `
+          --no-rate-limit `
+          --dev registry `
           --npseed 42 `
-          -k "not test_double_page_navigation and not test_handler_resource_management"
+          -k "not test_double_page_navigation and not test_handler_resource_management and not test_plugin_uninstall_ui and not test_sideloaded_script_lifecycle and not test_search_functionality and not test_ui_nofunmode_login_right_password and not test_ui_enable_nofunmode and not test_validation_carousel_search"
+      - name: Upload server logs
+        if: failure()
+        uses: actions/upload-artifact@v4
+        with:
+          name: server-logs-windows
+          path: ${{ github.workspace }}\server-logs\
+          retention-days: 14

.gitignore

@@ -1,3 +1,5 @@
+lib/LANraragi/Plugin/Managed
+
 node_modules
 content
 thumb

eslint.config.mjs

@@ -1,5 +1,5 @@
 import js from "@eslint/js";
-import { defineConfig } from "eslint/config";
+import { defineConfig, globalIgnores } from "eslint/config";
 import stylistic from "@stylistic/eslint-plugin";
 import globals from "globals";
 
@@ -9,7 +9,6 @@ import globals from "globals";
  */
 const config = {
     files: ["**/*.js"],
-    ignores: ["public/js/vendor/*.js"],
     plugins: {
         js,
         "@stylistic": stylistic,
@@ -22,27 +21,10 @@ const config = {
         globals: {
             ...globals.browser,
             ...globals.jquery,
-            // LANraragi specific
-            // TODO rework all main scripts to no longer store data in the global scope, probably transition to ES modules
-            Backup: "readonly",
-            Batch: "readonly",
-            Category: "readonly",
-            Common: "readonly",
-            Config: "readonly",
-            Duplicates: "readonly",
-            Edit: "readonly",
-            I18N: "readonly",
-            Index: "readonly",
-            IndexTable: "readonly",
-            Logs: "readonly",
-            LRR: "readonly",
-            Plugins: "readonly",
-            Reader: "readonly",
-            Server: "readonly",
-            Stats: "readonly",
             // external packages
             Awesomplete: "readonly",
-            marked: "readonly",
+            Raty: "readonly",
+            Sortable: "readonly",
             Swiper: "readonly",
             tagger: "readonly",
             tippy: "readonly",
@@ -84,4 +66,7 @@ const config = {
     },
 };
 
-export default defineConfig(config);
+export default defineConfig([
+    globalIgnores(["public/js/vendor/*.js", "tests/samples/*"]),
+    config,
+]);

lib/LANraragi.pm

@@ -23,6 +23,7 @@ use LANraragi::Utils::I18NInitializer;
 
 use LANraragi::Model::Search;
 use LANraragi::Model::Config;
+use LANraragi::Model::Plugins;
 use LANraragi::Model::Setup      qw(first_install_actions);
 use LANraragi::Model::Metrics;
 
@@ -134,6 +135,11 @@ sub startup {
     # through LRR's rotating logger pipeline.
     $self->log( get_logger( "Mojolicious", "mojo" ) );
 
+    # Reconcile discovered plugins with Redis state.
+    my $redis_config = $self->LRR_CONF->get_redis_config;
+    LANraragi::Model::Plugins::scan_plugins($redis_config);
+    $redis_config->quit();
+
     #Plugin listing
     my @plugins = get_plugins("metadata");
     foreach my $pluginfo (@plugins) {

lib/LANraragi/Controller/Api/Other.pm

@@ -1,12 +1,13 @@
 package LANraragi::Controller::Api::Other;
 use Mojo::Base 'Mojolicious::Controller';
 
-use Mojo::JSON qw(encode_json decode_json);
+use Mojo::JSON qw(encode_json decode_json true false);
 use Redis;
 
 use LANraragi::Model::Stats;
 use LANraragi::Model::Opds;
 use LANraragi::Utils::Generic    qw(render_api_response);
+use LANraragi::Utils::Logging    qw(get_logger);
 use LANraragi::Utils::Plugins    qw(get_plugin get_plugins use_plugin);
 
 sub serve_serverinfo {
@@ -90,6 +91,8 @@ sub list_plugins {
     my $type = $self->stash('type');
 
     my @plugins = get_plugins($type);
+    my $redis   = $self->LRR_CONF->get_redis_config;
+    my $logger  = get_logger( "Plugins", "lanraragi" );
 
     foreach my $plugin (@plugins) {
         if ( ref( $plugin->{parameters} ) eq 'HASH' ) {
@@ -99,8 +102,22 @@ sub list_plugins {
             }
             $plugin->{parameters} = \@parameters_array;
         }
+
+        my $namerds         = "LRR_PLUGIN_" . uc( $plugin->{namespace} );
+        $plugin->{registry} = $redis->hget( $namerds, "installed_registry" );
+        $plugin->{sha256}   = $redis->hget( $namerds, "installed_sha256" );
+
+        # Usually installed_version shouldn't be different from version.
+        my $installed_version   = $redis->hget( $namerds, "installed_version" );
+        my $plugin_version      = $plugin->{version};
+        if ( defined $installed_version && $installed_version ne $plugin_version ) {
+            $logger->warn(
+                "Plugin $plugin installed_version='$installed_version' but plugin->version='$plugin_version'"
+            );
+        }
     }
 
+    $redis->quit();
     $self->render( openapi => \@plugins );
 }
 

lib/LANraragi/Controller/Api/Plugins.pm

@@ -0,0 +1,130 @@
+package LANraragi::Controller::Api::Plugins;
+use Mojo::Base 'Mojolicious::Controller';
+
+use LANraragi::Model::Plugins;
+use LANraragi::Utils::Generic qw(render_api_response exec_with_lock);
+use LANraragi::Utils::Logging qw(get_logger);
+
+# Install a managed plugin.
+sub install_plugin {
+    my $self        = shift->openapi->valid_input or return;
+    my $body        = $self->req->json;
+    my $namespace   = $body->{namespace};
+    my $registry_id = $body->{registry};
+    my $version     = $body->{version};
+    my $force       = $body->{force} // 0;
+
+    # TODO: maybe consider extending TTL for this sub to 60s if it's not enough.
+    return unless exec_with_lock(
+        $self,
+        "plugin-write:" . uc($namespace),
+        "install_plugin",
+        $namespace,
+        sub {
+            my $redis = $self->LRR_CONF->get_redis_config;
+            # Since a plugin namespace can be built-in or managed, we'll need to check redis first.
+            # If a plugin exists and is not managed, installation may not continue.
+            # The user will have to remove/uninstall the existing plugin before installing a managed plugin
+            # by the same namespace.
+            my $namerds = "LRR_PLUGIN_" . uc($namespace);
+            if ( $redis->hexists( $namerds, "installed_path" ) ) {
+                my $source     = LANraragi::Model::Plugins::infer_plugin_origin( $namerds, $redis );
+                my $currentreg = $redis->hget( $namerds, "installed_registry" );
+                my $currentver = $redis->hget( $namerds, "installed_version" );
+
+                # only managed plugins can be upgraded.
+                if ( $source ne "managed" ) {
+                    $redis->quit();
+                    render_api_response(
+                        $self,
+                        "install_plugin",
+                        "Plugin '$namespace' already exists as a $source plugin. Remove it first before installing from a registry."
+                    );
+                    return;
+                }
+
+                # cross-registry overwrites require force installation.
+                my $is_cross_registry = $currentreg && $currentreg ne $registry_id;
+                if ( $is_cross_registry && !$force ) {
+                    $redis->quit();
+                    render_api_response(
+                        $self,
+                        "install_plugin",
+                        "Plugin '$namespace' already installed from '$currentreg' (v$currentver). Use force to overwrite."
+                    );
+                    return;
+                }
+            }
+
+            my $logger = get_logger( "Registry", "lanraragi" );
+            my $install_error;
+            my ( $status, $plugmeta, $message ) = eval {
+                LANraragi::Model::Plugins::install_plugin(
+                    $namespace, $redis, $registry_id, $version
+                );
+            };
+            $install_error = $@;
+
+            if ($install_error) {
+                $redis->quit();
+                $logger->error("install_plugin failed for '$namespace': $install_error");
+                render_api_response( $self, "install_plugin", "Plugin installation failed." );
+                return;
+            }
+
+            $redis->quit();
+
+            unless ( $status == 200 ) {
+                $self->render(
+                    openapi => { operation => "install_plugin", error => $message, success => 0 },
+                    status  => $status
+                );
+                return;
+            }
+
+            $self->render(
+                openapi => {
+                    operation          => "install_plugin",
+                    success            => 1,
+                    name               => $plugmeta->{name},
+                    namespace          => $namespace,
+                    version            => $plugmeta->{version},
+                    registry           => $plugmeta->{registry},
+                    sha256             => $plugmeta->{sha256},
+                }
+            );
+        },
+    );
+}
+
+# Uninstall a managed plugin.
+sub uninstall_plugin {
+    my $self      = shift->openapi->valid_input or return;
+    my $namespace = $self->stash('plugin_namespace');
+
+    return unless exec_with_lock(
+        $self,
+        "plugin-write:" . uc($namespace),
+        "uninstall_plugin",
+        $namespace,
+        sub {
+            my $redis = $self->LRR_CONF->get_redis_config;
+            my ( $status, $message ) = LANraragi::Model::Plugins::uninstall_plugin(
+                $namespace, $redis
+            );
+            $redis->quit();
+
+            unless ( $status == 200 ) {
+                $self->render(
+                    openapi => { operation => "uninstall_plugin", error => $message, success => 0 },
+                    status  => $status
+                );
+                return;
+            }
+
+            render_api_response( $self, "uninstall_plugin" );
+        }
+    );
+}
+
+1;