Schedule Sortinghat jobs using GrimoireLab core scheduler

README.md

@@ -179,6 +179,15 @@ and store it into SortingHat database.
 grimoirelab run ushers
 ```
 
+### Run SortingHat workers
+
+Run the SortingHat workers that will run jobs related to identities management.
+Like affiliate, unify, genderize, recommendations and identities importers.
+
+```
+grimoirelab run sortinghat-workers
+```
+
 #### Run the backend API
 
 Run the backend API server that will provide a REST API to manage the

releases/unreleased/sortinghat-tasks-and-updated-api.yml

@@ -0,0 +1,8 @@
+---
+title: SortingHat tasks and updated API
+category: added
+author: Jose Javier Merchante <jjmerchante@bitergia.com>
+issue: null
+notes: >
+  Include SortingHat tasks in GrimoireLab scheduler and extend the
+  API to support them alongside eventizer tasks.

src/grimoirelab/core/app/urls.py

@@ -13,14 +13,14 @@
 from sortinghat.core.views import SortingHatGraphQLView
 from ..views import api_login
 
-from grimoirelab.core.scheduler.urls import urlpatterns as sched_urlpatterns
+from grimoirelab.core.scheduler.urls import tasks_urlpatterns
 from grimoirelab.core.datasources.urls import ecosystems_urlpatterns
 
+
 urlpatterns = [
     path("login", api_login, name="api_login"),
     path("token/", TokenObtainPairView.as_view(), name="token_obtain_pair"),
     path("token/refresh/", TokenRefreshView.as_view(), name="token_refresh"),
-    path("scheduler/", include(sched_urlpatterns)),
     path(
         "api/v1/",
         include(
@@ -34,10 +34,13 @@
                         SortingHatGraphQLView.as_view(graphiql=settings.DEBUG, schema=schema)
                     ),
                 ),
+                # Tasks API
+                path("tasks/", include(tasks_urlpatterns)),
             ]
         ),
     ),
     re_path(
-        r"^(?!static|scheduler|datasources).*$", TemplateView.as_view(template_name="index.html")
+        r"^(?!static|login|token|api).*$",
+        TemplateView.as_view(template_name="index.html"),
     ),
 ]

src/grimoirelab/core/config/settings.py

@@ -293,6 +293,7 @@
 #
 
 GRIMOIRELAB_Q_EVENTIZER_JOBS = os.environ.get("GRIMOIRELAB_Q_EVENTIZER_JOBS", "eventizer_jobs")
+GRIMOIRELAB_Q_SORTINGHAT_JOBS = os.environ.get("GRIMOIRELAB_Q_SORTINGHAT_JOBS", "sortinghat_jobs")
 
 _RQ_DATABASE = {
     "HOST": os.environ.get("GRIMOIRELAB_REDIS_HOST", "127.0.0.1"),
@@ -304,6 +305,7 @@
 RQ_QUEUES = {
     "default": _RQ_DATABASE,
     GRIMOIRELAB_Q_EVENTIZER_JOBS: _RQ_DATABASE,
+    GRIMOIRELAB_Q_SORTINGHAT_JOBS: _RQ_DATABASE,
 }
 
 GRIMOIRELAB_EVENTS_STREAM_NAME = os.environ.get("GRIMOIRELAB_EVENTS_STREAM_NAME", "events")

src/grimoirelab/core/runner/commands/run.py

@@ -209,6 +209,36 @@ def eventizers(workers: int, verbose: bool, burst: bool):
     )
 
 
+@run.command()
+@worker_options(workers=5)
+def sortinghat_workers(workers: int, verbose: bool, burst: bool):
+    """Start a pool of SortingHat workers.
+
+    The workers on the pool will process SortingHat related tasks.
+
+    The number of workers running in the pool can be defined with the
+    parameter '--workers'.
+
+    To enable verbose mode, use the '--verbose' flag.
+
+    If the '--burst' flag is enabled, the pool will process all the events
+    and exit.
+
+    Workers get jobs from the GRIMOIRELAB_Q_SORTINGHAT_JOBS queue defined
+    in the configuration file.
+    """
+    _wait_redis_ready()
+    _wait_database_ready()
+
+    django.core.management.call_command(
+        "rqworker-pool",
+        settings.GRIMOIRELAB_Q_SORTINGHAT_JOBS,
+        num_workers=workers,
+        burst=burst,
+        verbosity=3 if verbose else 1,
+    )
+
+
 def _sleep_backoff(attempt: int) -> None:
     """Sleep with exponential backoff"""