diff --git a/releases/unreleased/api-endpoint-for-project-and-repository-events.yml b/releases/unreleased/api-endpoint-for-project-and-repository-events.yml new file mode 100644 index 0000000..ec098cb --- /dev/null +++ b/releases/unreleased/api-endpoint-for-project-and-repository-events.yml @@ -0,0 +1,9 @@ +--- +title: API endpoint for project and repository events +category: added +author: Jose Javier Merchante +issue: null +notes: > + Introduce two new API endpoints to fetch the latest events + for a specific project and repository. These endpoints allow + filtering by event type and date, and support pagination. diff --git a/src/grimoirelab/core/datasources/api.py b/src/grimoirelab/core/datasources/api.py index 21cf033..e904d73 100644 --- a/src/grimoirelab/core/datasources/api.py +++ b/src/grimoirelab/core/datasources/api.py @@ -24,6 +24,7 @@ response, serializers, status, + views, ) from drf_spectacular.utils import ( extend_schema, @@ -35,6 +36,8 @@ from django.db.models import Q from django.conf import settings from django.shortcuts import get_object_or_404 +from grimoirelab_toolkit.datetime import str_to_datetime, InvalidDateError +from rest_framework.exceptions import ValidationError from .models import ( DataSet, @@ -43,6 +46,7 @@ Project, ) from .utils import generate_uuid +from .events import get_events from ..scheduler.api import EventizerTaskSerializer from ..scheduler.scheduler import schedule_task, cancel_task @@ -287,7 +291,7 @@ def get_queryset(self): name=self.kwargs.get("project_name"), ecosystem__name=self.kwargs.get("ecosystem_name"), ) - queryset = Repository.objects.filter(dataset__project=project).distinct() + queryset = Repository.objects.filter(dataset__project=project).distinct().order_by("pk") datasource = self.request.query_params.get("datasource_type") category = self.request.query_params.get("category") @@ -512,3 +516,135 @@ def get_serializer_context(self): context = super().get_serializer_context() context.update({"project_id": self.project.id}) return context + + +class ProjectEventList(views.APIView): + """API endpoint that allows to get the latest events for a given project.""" + + def get(self, request, ecosystem_name, project_name): + event_type = request.query_params.get("type", None) + from_date = request.query_params.get("from_date", None) + to_date = request.query_params.get("to_date", None) + page = request.query_params.get("page", 1) + size = request.query_params.get("size", 25) + + # Validate page and size parameters + try: + page = int(page) + size = int(size) + except ValueError: + raise ValidationError("Page must be an integer.") + if page < 1: + raise ValidationError("Page must be greater than 0.") + if size < 1 or size > 100: + raise ValidationError("Size must be between 1 and 100.") + + # Parse from_date and to_date + from_date_parsed = None + to_date_parsed = None + try: + if from_date: + from_date_parsed = str_to_datetime(from_date) + if to_date: + to_date_parsed = str_to_datetime(to_date) + except InvalidDateError: + raise ValidationError("from_date and to_date must be in a valid datetime format.") + + # Obtain the repository sources for the given project + project = get_object_or_404( + Project, + name=project_name, + ecosystem__name=ecosystem_name, + ) + queryset = ( + Repository.objects.filter(dataset__project=project) + .distinct() + .values_list("uri", flat=True) + ) + sources = list(queryset) + + events = get_events( + sources=sources, + event_type=event_type, + from_date=from_date_parsed, + to_date=to_date_parsed, + page=page, + size=size, + ) + total = events.hits.total.value + + return response.Response( + { + "count": total, + "page": page, + "total_pages": (total + size - 1) // size, + "results": [hit.to_dict() for hit in events], + }, + status=status.HTTP_200_OK, + ) + + +class RepoEventList(views.APIView): + """API endpoint that allows to get the latest events for a given repository.""" + + def get(self, request, ecosystem_name, project_name, uuid): + event_type = request.query_params.get("type", None) + from_date = request.query_params.get("from_date", None) + to_date = request.query_params.get("to_date", None) + page = request.query_params.get("page", 1) + size = request.query_params.get("size", 25) + + # Validate page and size parameters + try: + page = int(page) + size = int(size) + except ValueError: + raise ValidationError("Page must be an integer.") + if page < 1: + raise ValidationError("Page must be greater than 0.") + if size < 1 or size > 100: + raise ValidationError("Size must be between 1 and 100.") + + # Parse from_date and to_date + from_date_parsed = None + to_date_parsed = None + try: + if from_date: + from_date_parsed = str_to_datetime(from_date) + if to_date: + to_date_parsed = str_to_datetime(to_date) + except InvalidDateError: + raise ValidationError("from_date and to_date must be in a valid datetime format.") + + # Obtain the repository source for the given repository + project = get_object_or_404( + Project, + name=project_name, + ecosystem__name=ecosystem_name, + ) + repository = get_object_or_404( + Repository, + uuid=uuid, + dataset__project=project, + ) + source = repository.uri + + events = get_events( + sources=[source], + event_type=event_type, + from_date=from_date_parsed, + to_date=to_date_parsed, + page=page, + size=size, + ) + total = events.hits.total.value + + return response.Response( + { + "count": total, + "page": page, + "total_pages": (total + size - 1) // size, + "results": [hit.to_dict() for hit in events], + }, + status=status.HTTP_200_OK, + ) diff --git a/src/grimoirelab/core/datasources/events.py b/src/grimoirelab/core/datasources/events.py new file mode 100644 index 0000000..82fbb32 --- /dev/null +++ b/src/grimoirelab/core/datasources/events.py @@ -0,0 +1,52 @@ +import datetime + +from opensearchpy import Search + +from ..config import settings +from ..utils.opensearch import get_opensearch_client + + +def get_events( + sources: list = None, + event_type: str = None, + from_date: datetime.datetime = None, + to_date: datetime.datetime = None, + page: int = 1, + size: int = 25, +) -> list: + """ + Retrieve events from OpenSearch with optional filtering and pagination. + + :param sources: List of repository sources to filter by. + :param event_type: Type of event to filter by. + :param from_date: Start date for filtering events. + :param to_date: End date for filtering events. + :param page: Page number for pagination. + :param size: Number of events per page. + :return: OpenSearch response object containing the events. + """ + opensearch = get_opensearch_client() + + index = settings.GRIMOIRELAB_ARCHIVIST["STORAGE_INDEX"] + s = Search(using=opensearch, index=index) + s = s.sort({"time": {"order": "asc"}}, {"id": {"order": "asc"}}) + + if sources: + s = s.filter("terms", source=sources) + + if event_type: + s = s.filter("term", type=event_type) + + if from_date or to_date: + range_filter = {} + if from_date: + range_filter["gte"] = from_date + if to_date: + range_filter["lte"] = to_date + s = s.filter("range", time=range_filter) + + s = s[(page - 1) * size : page * size] + + response = s.execute() + + return response diff --git a/src/grimoirelab/core/datasources/urls.py b/src/grimoirelab/core/datasources/urls.py index 60d2a46..b3910ff 100644 --- a/src/grimoirelab/core/datasources/urls.py +++ b/src/grimoirelab/core/datasources/urls.py @@ -30,6 +30,11 @@ api.ProjectDetail.as_view(), name="projects-detail", ), + path( + "/projects//events/", + api.ProjectEventList.as_view(), + name="project-events", + ), path( "/projects//children/", api.ProjectChildrenList.as_view(), @@ -45,6 +50,11 @@ api.RepoDetail.as_view(), name="repo-detail", ), + path( + "/projects//repos//events/", + api.RepoEventList.as_view(), + name="repo-events", + ), path( "/projects//repos//categories//", api.CategoryDetail.as_view(), diff --git a/src/grimoirelab/core/runner/commands/run.py b/src/grimoirelab/core/runner/commands/run.py index 31db20e..3139664 100644 --- a/src/grimoirelab/core/runner/commands/run.py +++ b/src/grimoirelab/core/runner/commands/run.py @@ -24,7 +24,6 @@ import time import typing -import certifi import click import django.core import django.core.wsgi @@ -35,7 +34,8 @@ from django.conf import settings from django.db import connections, OperationalError -from urllib3.util import create_urllib3_context + +from grimoirelab.core.utils.opensearch import get_opensearch_client if typing.TYPE_CHECKING: from click import Context @@ -216,9 +216,7 @@ def _sleep_backoff(attempt: int) -> None: time.sleep(backoff) -def _wait_opensearch_ready( - url: str, username: str | None, password: str | None, index: str, verify_certs: bool -) -> None: +def _wait_opensearch_ready(index: str) -> None: """Wait for OpenSearch to be available before starting""" # The 'opensearch' library writes logs with the exceptions while @@ -228,25 +226,10 @@ def _wait_opensearch_ready( os_logger = logging.getLogger("opensearch") os_logger.disabled = True - context = None - if verify_certs: - # Use certificates from the local system and certifi - context = create_urllib3_context() - context.load_default_certs() - context.load_verify_locations(certifi.where()) - - auth = (username, password) if username and password else None + client = get_opensearch_client() for attempt in range(DEFAULT_MAX_RETRIES): try: - client = opensearchpy.OpenSearch( - hosts=[url], - http_auth=auth, - http_compress=True, - verify_certs=verify_certs, - ssl_context=context, - ssl_show_warn=False, - ) client.search(index=index, size=0) break except opensearchpy.exceptions.NotFoundError: @@ -341,11 +324,7 @@ def archivists(workers: int, verbose: bool, burst: bool): from grimoirelab.core.consumers.archivist import OpenSearchArchivistPool _wait_opensearch_ready( - settings.GRIMOIRELAB_ARCHIVIST["STORAGE_URL"], - settings.GRIMOIRELAB_ARCHIVIST["STORAGE_USERNAME"], - settings.GRIMOIRELAB_ARCHIVIST["STORAGE_PASSWORD"], settings.GRIMOIRELAB_ARCHIVIST["STORAGE_INDEX"], - settings.GRIMOIRELAB_ARCHIVIST["STORAGE_VERIFY_CERT"], ) _wait_redis_ready() diff --git a/src/grimoirelab/core/utils/__init__.py b/src/grimoirelab/core/utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/grimoirelab/core/utils/opensearch.py b/src/grimoirelab/core/utils/opensearch.py new file mode 100644 index 0000000..5f91b18 --- /dev/null +++ b/src/grimoirelab/core/utils/opensearch.py @@ -0,0 +1,58 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) GrimoireLab Contributors +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + +import warnings + +import certifi +import urllib3 +from opensearchpy import OpenSearch +from django.conf import settings +from urllib3.util import create_urllib3_context + + +def get_opensearch_client(): + url = settings.GRIMOIRELAB_ARCHIVIST["STORAGE_URL"] + username = settings.GRIMOIRELAB_ARCHIVIST["STORAGE_USERNAME"] + password = settings.GRIMOIRELAB_ARCHIVIST["STORAGE_PASSWORD"] + verify_certs = settings.GRIMOIRELAB_ARCHIVIST["STORAGE_VERIFY_CERT"] + + context = None + if verify_certs: + # Use certificates from the local system and certifi + context = create_urllib3_context() + context.load_default_certs() + context.load_verify_locations(certifi.where()) + else: + # Ignore SSL warnings if not verifying certificates + urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) + warnings.filterwarnings("ignore", message=".*verify_certs.*") + + auth = (username, password) if username and password else None + + client = OpenSearch( + hosts=[url], + http_auth=auth, + http_compress=True, + verify_certs=verify_certs, + ssl_context=context, + ssl_show_warn=False, + max_retries=3, + retry_on_timeout=True, + ) + + return client diff --git a/tests/unit/datasources/test_api.py b/tests/unit/datasources/test_api.py index b7e53c9..e402171 100644 --- a/tests/unit/datasources/test_api.py +++ b/tests/unit/datasources/test_api.py @@ -16,7 +16,7 @@ # along with this program. If not, see . # -from unittest.mock import patch +from unittest.mock import patch, MagicMock from django.contrib.auth import get_user_model from django.core.exceptions import ObjectDoesNotExist @@ -1387,3 +1387,123 @@ def test_unauthenticated_request(self): response = self.client.get(url) self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN) self.assertEqual(response.data["detail"], "Authentication credentials were not provided.") + + +class RepoProjectEventsApiTest(APITestCase): + """Unit tests for the RepoEventList and ProjectEventList API endpoints""" + + def setUp(self): + user = get_user_model().objects.create(username="test", is_superuser=True) + self.client.force_authenticate(user=user) + self.ecosystem = Ecosystem.objects.create(name="ecosystem1", title="Ecosystem 1") + self.project = Project.objects.create( + name="example-project", title="Example Project", ecosystem=self.ecosystem + ) + self.repository = Repository.objects.create( + uuid="AAA", uri="https://example.com/repo.git", datasource_type="git" + ) + DataSet.objects.create( + project=self.project, repository=self.repository, category="category1" + ) + + def _make_mock_events(self, count, total): + """Helper method to create a list of mock events""" + + mock_events = MagicMock() + mock_events.hits = MagicMock() + mock_events.hits.total = MagicMock() + mock_events.hits.total.value = total + + mock_events.__iter__.return_value = [ + MagicMock(to_dict=MagicMock(return_value={"id": f"event-{i}"})) for i in range(count) + ] + + return mock_events + + @patch("grimoirelab.core.datasources.api.get_events") + def test_repo_events_success(self, mock_get_events): + """Test successful retrieval of repo events""" + + mock_events = self._make_mock_events(count=5, total=10) + mock_get_events.return_value = mock_events + + url = reverse( + "repo-events", + kwargs={ + "ecosystem_name": self.ecosystem.name, + "project_name": self.project.name, + "uuid": self.repository.uuid, + }, + ) + response = self.client.get(url, {"page": 1, "size": 5, "from_date": "2023-01-01"}) + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(response.data["count"], 10) + self.assertEqual(response.data["page"], 1) + self.assertEqual(response.data["total_pages"], 2) + self.assertEqual(len(response.data["results"]), 5) + + for i in range(5): + self.assertEqual(response.data["results"][i]["id"], f"event-{i}") + + def test_repo_events_invalid_page_and_size(self): + """Test validation errors for non-integer page/size and out-of-range size""" + + url = reverse( + "repo-events", + kwargs={ + "ecosystem_name": self.ecosystem.name, + "project_name": self.project.name, + "uuid": self.repository.uuid, + }, + ) + + # Non-integer page + response = self.client.get(url, {"page": "not-an-int"}) + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + + # Page less than 1 + response = self.client.get(url, {"page": 0}) + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + + # Size out of bounds (too large) + response = self.client.get(url, {"size": 101}) + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + + def test_repo_events_invalid_date(self): + """Test that invalid date formats raise a validation error""" + + url = reverse( + "repo-events", + kwargs={ + "ecosystem_name": self.ecosystem.name, + "project_name": self.project.name, + "uuid": self.repository.uuid, + }, + ) + + response = self.client.get(url, {"from_date": "invalid-date-format"}) + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + + @patch("grimoirelab.core.datasources.api.get_events") + def test_project_events_success(self, mock_get_events): + """Test successful retrieval of project events""" + + mock_events = self._make_mock_events(count=5, total=10) + mock_get_events.return_value = mock_events + + url = reverse( + "project-events", + kwargs={ + "ecosystem_name": self.ecosystem.name, + "project_name": self.project.name, + }, + ) + response = self.client.get(url, {"page": 1, "size": 5, "from_date": "2023-01-01"}) + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(response.data["count"], 10) + self.assertEqual(response.data["page"], 1) + self.assertEqual(response.data["total_pages"], 2) + self.assertEqual(len(response.data["results"]), 5) + + for i in range(5): + self.assertEqual(response.data["results"][i]["id"], f"event-{i}")