Add events API endpoint

releases/unreleased/api-endpoint-for-project-and-repository-events.yml

@@ -0,0 +1,9 @@
+---
+title: API endpoint for project and repository events
+category: added
+author: Jose Javier Merchante <jjmerchante@bitergia.com>
+issue: null
+notes: >
+  Introduce two new API endpoints to fetch the latest events
+  for a specific project and repository. These endpoints allow
+  filtering by event type and date, and support pagination.

src/grimoirelab/core/datasources/api.py

@@ -24,6 +24,7 @@
     response,
     serializers,
     status,
+    views,
 )
 from drf_spectacular.utils import (
     extend_schema,
@@ -35,6 +36,8 @@
 from django.db.models import Q
 from django.conf import settings
 from django.shortcuts import get_object_or_404
+from grimoirelab_toolkit.datetime import str_to_datetime, InvalidDateError
+from rest_framework.exceptions import ValidationError
 
 from .models import (
     DataSet,
@@ -43,6 +46,7 @@
     Project,
 )
 from .utils import generate_uuid
+from .events import get_events
 from ..scheduler.api import EventizerTaskSerializer
 from ..scheduler.scheduler import schedule_task, cancel_task
 
@@ -287,7 +291,7 @@ def get_queryset(self):
             name=self.kwargs.get("project_name"),
             ecosystem__name=self.kwargs.get("ecosystem_name"),
         )
-        queryset = Repository.objects.filter(dataset__project=project).distinct()
+        queryset = Repository.objects.filter(dataset__project=project).distinct().order_by("pk")
 
         datasource = self.request.query_params.get("datasource_type")
         category = self.request.query_params.get("category")
@@ -512,3 +516,135 @@ def get_serializer_context(self):
         context = super().get_serializer_context()
         context.update({"project_id": self.project.id})
         return context
+
+
+class ProjectEventList(views.APIView):
+    """API endpoint that allows to get the latest events for a given project."""
+
+    def get(self, request, ecosystem_name, project_name):
+        event_type = request.query_params.get("type", None)
+        from_date = request.query_params.get("from_date", None)
+        to_date = request.query_params.get("to_date", None)
+        page = request.query_params.get("page", 1)
+        size = request.query_params.get("size", 25)
+
+        # Validate page and size parameters
+        try:
+            page = int(page)
+            size = int(size)
+        except ValueError:
+            raise ValidationError("Page must be an integer.")
+        if page < 1:
+            raise ValidationError("Page must be greater than 0.")
+        if size < 1 or size > 100:
+            raise ValidationError("Size must be between 1 and 100.")
+
+        # Parse from_date and to_date
+        from_date_parsed = None
+        to_date_parsed = None
+        try:
+            if from_date:
+                from_date_parsed = str_to_datetime(from_date)
+            if to_date:
+                to_date_parsed = str_to_datetime(to_date)
+        except InvalidDateError:
+            raise ValidationError("from_date and to_date must be in a valid datetime format.")
+
+        # Obtain the repository sources for the given project
+        project = get_object_or_404(
+            Project,
+            name=project_name,
+            ecosystem__name=ecosystem_name,
+        )
+        queryset = (
+            Repository.objects.filter(dataset__project=project)
+            .distinct()
+            .values_list("uri", flat=True)
+        )
+        sources = list(queryset)
+
+        events = get_events(
+            sources=sources,
+            event_type=event_type,
+            from_date=from_date_parsed,
+            to_date=to_date_parsed,
+            page=page,
+            size=size,
+        )
+        total = events.hits.total.value
+
+        return response.Response(
+            {
+                "count": total,
+                "page": page,
+                "total_pages": (total + size - 1) // size,
+                "results": [hit.to_dict() for hit in events],
+            },
+            status=status.HTTP_200_OK,
+        )
+
+
+class RepoEventList(views.APIView):
+    """API endpoint that allows to get the latest events for a given repository."""
+
+    def get(self, request, ecosystem_name, project_name, uuid):
+        event_type = request.query_params.get("type", None)
+        from_date = request.query_params.get("from_date", None)
+        to_date = request.query_params.get("to_date", None)
+        page = request.query_params.get("page", 1)
+        size = request.query_params.get("size", 25)
+
+        # Validate page and size parameters
+        try:
+            page = int(page)
+            size = int(size)
+        except ValueError:
+            raise ValidationError("Page must be an integer.")
+        if page < 1:
+            raise ValidationError("Page must be greater than 0.")
+        if size < 1 or size > 100:
+            raise ValidationError("Size must be between 1 and 100.")
+
+        # Parse from_date and to_date
+        from_date_parsed = None
+        to_date_parsed = None
+        try:
+            if from_date:
+                from_date_parsed = str_to_datetime(from_date)
+            if to_date:
+                to_date_parsed = str_to_datetime(to_date)
+        except InvalidDateError:
+            raise ValidationError("from_date and to_date must be in a valid datetime format.")
+
+        # Obtain the repository source for the given repository
+        project = get_object_or_404(
+            Project,
+            name=project_name,
+            ecosystem__name=ecosystem_name,
+        )
+        repository = get_object_or_404(
+            Repository,
+            uuid=uuid,
+            dataset__project=project,
+        )
+        source = repository.uri
+
+        events = get_events(
+            sources=[source],
+            event_type=event_type,
+            from_date=from_date_parsed,
+            to_date=to_date_parsed,
+            page=page,
+            size=size,
+        )
+        total = events.hits.total.value
+
+        return response.Response(
+            {
+                "count": total,
+                "page": page,
+                "total_pages": (total + size - 1) // size,
+                "results": [hit.to_dict() for hit in events],
+            },
+            status=status.HTTP_200_OK,
+        )

src/grimoirelab/core/datasources/events.py

@@ -0,0 +1,52 @@
+import datetime
+
+from opensearchpy import Search
+
+from ..config import settings
+from ..utils.opensearch import get_opensearch_client
+
+
+def get_events(
+    sources: list = None,
+    event_type: str = None,
+    from_date: datetime.datetime = None,
+    to_date: datetime.datetime = None,
+    page: int = 1,
+    size: int = 25,
+) -> list:
+    """
+    Retrieve events from OpenSearch with optional filtering and pagination.
+
+    :param sources: List of repository sources to filter by.
+    :param event_type: Type of event to filter by.
+    :param from_date: Start date for filtering events.
+    :param to_date: End date for filtering events.
+    :param page: Page number for pagination.
+    :param size: Number of events per page.
+    :return: OpenSearch response object containing the events.
+    """
+    opensearch = get_opensearch_client()
+
+    index = settings.GRIMOIRELAB_ARCHIVIST["STORAGE_INDEX"]
+    s = Search(using=opensearch, index=index)
+    s = s.sort({"time": {"order": "asc"}}, {"id": {"order": "asc"}})
+
+    if sources:
+        s = s.filter("terms", source=sources)
+
+    if event_type:
+        s = s.filter("term", type=event_type)
+
+    if from_date or to_date:
+        range_filter = {}
+        if from_date:
+            range_filter["gte"] = from_date
+        if to_date:
+            range_filter["lte"] = to_date
+        s = s.filter("range", time=range_filter)
+
+    s = s[(page - 1) * size : page * size]
+
+    response = s.execute()
+
+    return response

src/grimoirelab/core/datasources/urls.py

@@ -30,6 +30,11 @@
         api.ProjectDetail.as_view(),
         name="projects-detail",
     ),
+    path(
+        "<str:ecosystem_name>/projects/<str:project_name>/events/",
+        api.ProjectEventList.as_view(),
+        name="project-events",
+    ),
     path(
         "<str:ecosystem_name>/projects/<str:project_name>/children/",
         api.ProjectChildrenList.as_view(),
@@ -45,6 +50,11 @@
         api.RepoDetail.as_view(),
         name="repo-detail",
     ),
+    path(
+        "<str:ecosystem_name>/projects/<str:project_name>/repos/<str:uuid>/events/",
+        api.RepoEventList.as_view(),
+        name="repo-events",
+    ),
     path(
         "<str:ecosystem_name>/projects/<str:project_name>/repos/<str:uuid>/categories/<str:category>/",
         api.CategoryDetail.as_view(),

src/grimoirelab/core/runner/commands/run.py

@@ -24,7 +24,6 @@
 import time
 import typing
 
-import certifi
 import click
 import django.core
 import django.core.wsgi
@@ -35,7 +34,8 @@
 
 from django.conf import settings
 from django.db import connections, OperationalError
-from urllib3.util import create_urllib3_context
+
+from grimoirelab.core.utils.opensearch import get_opensearch_client
 
 if typing.TYPE_CHECKING:
     from click import Context
@@ -216,9 +216,7 @@ def _sleep_backoff(attempt: int) -> None:
     time.sleep(backoff)
 
 
-def _wait_opensearch_ready(
-    url: str, username: str | None, password: str | None, index: str, verify_certs: bool
-) -> None:
+def _wait_opensearch_ready(index: str) -> None:
     """Wait for OpenSearch to be available before starting"""
 
     # The 'opensearch' library writes logs with the exceptions while
@@ -228,25 +226,10 @@ def _wait_opensearch_ready(
     os_logger = logging.getLogger("opensearch")
     os_logger.disabled = True
 
-    context = None
-    if verify_certs:
-        # Use certificates from the local system and certifi
-        context = create_urllib3_context()
-        context.load_default_certs()
-        context.load_verify_locations(certifi.where())
-
-    auth = (username, password) if username and password else None
+    client = get_opensearch_client()
 
     for attempt in range(DEFAULT_MAX_RETRIES):
         try:
-            client = opensearchpy.OpenSearch(
-                hosts=[url],
-                http_auth=auth,
-                http_compress=True,
-                verify_certs=verify_certs,
-                ssl_context=context,
-                ssl_show_warn=False,
-            )
             client.search(index=index, size=0)
             break
         except opensearchpy.exceptions.NotFoundError:
@@ -341,11 +324,7 @@ def archivists(workers: int, verbose: bool, burst: bool):
     from grimoirelab.core.consumers.archivist import OpenSearchArchivistPool
 
     _wait_opensearch_ready(
-        settings.GRIMOIRELAB_ARCHIVIST["STORAGE_URL"],
-        settings.GRIMOIRELAB_ARCHIVIST["STORAGE_USERNAME"],
-        settings.GRIMOIRELAB_ARCHIVIST["STORAGE_PASSWORD"],
         settings.GRIMOIRELAB_ARCHIVIST["STORAGE_INDEX"],
-        settings.GRIMOIRELAB_ARCHIVIST["STORAGE_VERIFY_CERT"],
     )
     _wait_redis_ready()
 

src/grimoirelab/core/utils/opensearch.py

@@ -0,0 +1,58 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) GrimoireLab Contributors
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+
+import warnings
+
+import certifi
+import urllib3
+from opensearchpy import OpenSearch
+from django.conf import settings
+from urllib3.util import create_urllib3_context
+
+
+def get_opensearch_client():
+    url = settings.GRIMOIRELAB_ARCHIVIST["STORAGE_URL"]
+    username = settings.GRIMOIRELAB_ARCHIVIST["STORAGE_USERNAME"]
+    password = settings.GRIMOIRELAB_ARCHIVIST["STORAGE_PASSWORD"]
+    verify_certs = settings.GRIMOIRELAB_ARCHIVIST["STORAGE_VERIFY_CERT"]
+
+    context = None
+    if verify_certs:
+        # Use certificates from the local system and certifi
+        context = create_urllib3_context()
+        context.load_default_certs()
+        context.load_verify_locations(certifi.where())
+    else:
+        # Ignore SSL warnings if not verifying certificates
+        urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
+        warnings.filterwarnings("ignore", message=".*verify_certs.*")
+
+    auth = (username, password) if username and password else None
+
+    client = OpenSearch(
+        hosts=[url],
+        http_auth=auth,
+        http_compress=True,
+        verify_certs=verify_certs,
+        ssl_context=context,
+        ssl_show_warn=False,
+        max_retries=3,
+        retry_on_timeout=True,
+    )
+
+    return client