Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 13 additions & 0 deletions src/grimoirelab/core/consumers/identities.py
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,14 @@
GIT_EVENT_COMMIT_TESTED_BY,
)

from chronicler.events.core.github import (
GITHUB_EVENT_AUTHOR,
GITHUB_EVENT_ASSIGNEE,
GITHUB_EVENT_CLOSED_BY,
GITHUB_EVENT_CREATED_BY,
GITHUB_EVENT_COMMENT_AUTHOR,
)


IDENTITY_EVENTS = (
GIT_EVENT_COMMIT_AUTHORED_BY,
Expand All @@ -58,6 +66,11 @@
GIT_EVENT_COMMIT_SIGNED_OFF_BY,
GIT_EVENT_COMMIT_SUGGESTED_BY,
GIT_EVENT_COMMIT_TESTED_BY,
GITHUB_EVENT_AUTHOR,
GITHUB_EVENT_ASSIGNEE,
GITHUB_EVENT_CLOSED_BY,
GITHUB_EVENT_CREATED_BY,
GITHUB_EVENT_COMMENT_AUTHOR,
)


Expand Down
25 changes: 19 additions & 6 deletions src/grimoirelab/core/scheduler/tasks/chronicler.py
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,12 @@ def chronicler_job(

backend_args = job_args.copy() if job_args else {}

# Convert common datetime arguments
datetime_args = ["from_date", "to_date"]
for arg in datetime_args:
if arg in backend_args and isinstance(backend_args[arg], str):
backend_args[arg] = str_to_datetime(backend_args[arg])

# Get the generator to fetch the data items
perceval_gen = perceval.backend.BackendItemsGenerator(
backend_class, backend_args, datasource_category
Expand Down Expand Up @@ -222,7 +228,7 @@ def resuming_args(
params = {}

if progress.summary and progress.summary.fetched > 0:
params["from_date"] = progress.summary.max_updated_on
params["from_date"] = progress.summary.max_updated_on.isoformat()

if progress.summary.max_offset:
params["offset"] = progress.summary.max_offset
Expand All @@ -239,7 +245,7 @@ def recovery_args(
return ChroniclerArgumentGenerator.resuming_args(task_args, progress)


def get_chronicler_argument_generator(name: str) -> ChroniclerArgumentGenerator:
def get_chronicler_argument_generator(name: str) -> type[ChroniclerArgumentGenerator]:
"""Get the argument generator for a backend."""

generators = {
Expand Down Expand Up @@ -320,9 +326,14 @@ def initial_args(task_args: dict[str, Any]) -> dict[str, Any]:

# For the first execution make some arguments mandatory
job_args = {}
job_args["owner"] = task_args["owner"]
job_args["repository"] = task_args["repository"]

uri = task_args["uri"]
processed_uri = uri.rstrip("/").rstrip(".git", "")
path_parts = processed_uri.split("/")
job_args["owner"] = path_parts[-2]
job_args["repository"] = path_parts[-1]

# TODO: Obtain tokens from a secure storage
tokens = task_args.get("api_token", [])

if not isinstance(tokens, list):
Expand All @@ -331,6 +342,8 @@ def initial_args(task_args: dict[str, Any]) -> dict[str, Any]:
job_args["api_token"] = tokens
job_args["sleep_for_rate"] = True

return job_args

@staticmethod
def resuming_args(
task_args: dict[str, Any] | None,
Expand All @@ -340,7 +353,7 @@ def resuming_args(

job_args = task_args.copy() if task_args else {}
job_args["sleep_for_rate"] = True
job_args["from_date"] = progress.summary.last_updated_on
job_args["from_date"] = progress.summary.last_updated_on.isoformat()

return job_args

Expand All @@ -354,7 +367,7 @@ def recovery_args(
job_args = task_args.copy() if task_args else {}

if progress.summary and progress.summary.last_updated_on:
job_args["from_date"] = progress.summary.last_updated_on
job_args["from_date"] = progress.summary.last_updated_on.isoformat()

return job_args

Expand Down
Loading
Loading