diff --git a/packages/das/src/api/api.module.ts b/packages/das/src/api/api.module.ts index 2f8dea2..c6cb2a0 100644 --- a/packages/das/src/api/api.module.ts +++ b/packages/das/src/api/api.module.ts @@ -13,10 +13,14 @@ import { FETCH_QUEUE } from "../queue/constants"; import { AdminController } from "./admin.controller"; import { RequireApiKeyGuard } from "./require-api-key.guard"; import { HealthController } from "./health.controller"; +import { DashboardController } from "./dashboard/dashboard.controller"; +import { DashboardService } from "./dashboard/dashboard.service"; import { MinersController } from "./miners/miners.controller"; import { MinersService } from "./miners/miners.service"; import { PullsController } from "./pulls/pulls.controller"; import { PullsService } from "./pulls/pulls.service"; +import { ReposController } from "./repos/repos.controller"; +import { ReposService } from "./repos/repos.service"; @Module({ imports: [ @@ -31,11 +35,19 @@ import { PullsService } from "./pulls/pulls.service"; BullModule.registerQueue({ name: FETCH_QUEUE }), ], controllers: [ + DashboardController, MinersController, PullsController, + ReposController, AdminController, HealthController, ], - providers: [MinersService, PullsService, RequireApiKeyGuard], + providers: [ + DashboardService, + MinersService, + PullsService, + ReposService, + RequireApiKeyGuard, + ], }) export class ApiModule {} diff --git a/packages/das/src/api/dashboard/dashboard.controller.ts b/packages/das/src/api/dashboard/dashboard.controller.ts new file mode 100644 index 0000000..f332206 --- /dev/null +++ b/packages/das/src/api/dashboard/dashboard.controller.ts @@ -0,0 +1,37 @@ +import { BadRequestException, Controller, Get, Query } from "@nestjs/common"; +import { ApiOperation, ApiQuery, ApiTags } from "@nestjs/swagger"; +import { DashboardService } from "./dashboard.service"; + +@ApiTags("Dashboard") +@Controller("api/v1/dashboard") +export class DashboardController { + constructor(private readonly dashboard: DashboardService) {} + + @Get("issues") + @ApiOperation({ + summary: "Slim issue rows for dashboard trend aggregation", + description: + "Returns every issue with `created_at` on or after `since`, plus " + + "every CLOSED issue whose `closed_at` is on or after `since`. " + + "The mirror is intentionally roster-blind: every issue is returned " + + "regardless of author. The dashboard blends with the gittensor API " + + "miner roster client-side to filter to subnet authors. " + + "Designed as a single bulk replacement for the dashboard's per-miner " + + "fan-out against `/miners//issues` (one call instead of N).", + }) + @ApiQuery({ + name: "since", + required: true, + description: "ISO timestamp — earliest creation/close date to include.", + }) + async getIssues(@Query("since") since?: string): Promise { + if (!since) { + throw new BadRequestException("`since` query parameter is required"); + } + const parsed = new Date(since); + if (Number.isNaN(parsed.getTime())) { + throw new BadRequestException("`since` must be a valid ISO timestamp"); + } + return this.dashboard.getIssues(parsed.toISOString()); + } +} diff --git a/packages/das/src/api/dashboard/dashboard.service.ts b/packages/das/src/api/dashboard/dashboard.service.ts new file mode 100644 index 0000000..62a7ab6 --- /dev/null +++ b/packages/das/src/api/dashboard/dashboard.service.ts @@ -0,0 +1,58 @@ +/* eslint-disable @typescript-eslint/no-unsafe-return, @typescript-eslint/no-unsafe-assignment */ +import { Injectable } from "@nestjs/common"; +import { DataSource } from "typeorm"; + +interface DashboardIssueRow { + repo_full_name: string; + issue_number: number; + author_github_id: string | null; + created_at: string; + closed_at: string | null; + state: string; + state_reason: string | null; + solving_pr: { merged_at: string } | null; +} + +@Injectable() +export class DashboardService { + constructor(private readonly dataSource: DataSource) {} + + async getIssues(since: string): Promise<{ + since: string; + generated_at: string; + issues: DashboardIssueRow[]; + }> { + const rows = await this.dataSource.query( + ` + SELECT + LOWER(i.repo_full_name) AS repo_full_name, + i.issue_number, + i.author_github_id, + i.created_at, + i.closed_at, + i.state, + i.state_reason, + ( + SELECT json_build_object('merged_at', sp.merged_at) + FROM pull_requests sp + WHERE sp.repo_full_name = i.repo_full_name + AND sp.pr_number = i.solved_by_pr + AND sp.merged_at IS NOT NULL + LIMIT 1 + ) AS solving_pr + FROM issues i + WHERE + i.created_at >= $1 + OR (i.state = 'CLOSED' AND i.closed_at >= $1) + ORDER BY i.created_at DESC + `, + [since], + ); + + return { + since, + generated_at: new Date().toISOString(), + issues: rows as DashboardIssueRow[], + }; + } +} diff --git a/packages/das/src/api/repos/repos.controller.ts b/packages/das/src/api/repos/repos.controller.ts new file mode 100644 index 0000000..09314a7 --- /dev/null +++ b/packages/das/src/api/repos/repos.controller.ts @@ -0,0 +1,27 @@ +import { Controller, Get, Param } from "@nestjs/common"; +import { ApiOperation, ApiParam, ApiTags } from "@nestjs/swagger"; +import { ReposService } from "./repos.service"; + +@ApiTags("Repos") +@Controller("api/v1/repos") +export class ReposController { + constructor(private readonly repos: ReposService) {} + + @Get(":owner/:repo/maintainers") + @ApiOperation({ + summary: "Maintainer-role contributors for a repo", + description: + "Returns users whose latest known GitHub association for the repo " + + "is OWNER, MEMBER, or COLLABORATOR, synthesized from PR/issue/" + + "review/comment activity (contributor_repo_roles view). An unknown " + + "repo returns an empty maintainers list, not a 404.", + }) + @ApiParam({ name: "owner", description: "Repository owner (org or user)" }) + @ApiParam({ name: "repo", description: "Repository name" }) + async getMaintainers( + @Param("owner") owner: string, + @Param("repo") repo: string, + ): Promise { + return this.repos.getMaintainers(owner, repo); + } +} diff --git a/packages/das/src/api/repos/repos.service.ts b/packages/das/src/api/repos/repos.service.ts new file mode 100644 index 0000000..18e3b9c --- /dev/null +++ b/packages/das/src/api/repos/repos.service.ts @@ -0,0 +1,41 @@ +/* eslint-disable @typescript-eslint/no-unsafe-return, @typescript-eslint/no-unsafe-assignment */ +import { Injectable } from "@nestjs/common"; +import { DataSource } from "typeorm"; + +@Injectable() +export class ReposService { + constructor(private readonly dataSource: DataSource) {} + + async getMaintainers( + owner: string, + repo: string, + ): Promise<{ + repo_full_name: string; + generated_at: string; + maintainers: unknown[]; + }> { + const repoFullName = `${owner}/${repo}`; + + // The association literals must stay in sync with gittensor + // constants.py MAINTAINER_ASSOCIATIONS. + const rows = await this.dataSource.query( + ` + SELECT + cr.author_github_id AS github_id, + cr.author_login AS login, + cr.author_association AS association + FROM contributor_repo_roles cr + WHERE LOWER(cr.repo_full_name) = LOWER($1) + AND cr.author_association IN ('OWNER', 'MEMBER', 'COLLABORATOR') + ORDER BY cr.author_github_id + `, + [repoFullName], + ); + + return { + repo_full_name: repoFullName.toLowerCase(), + generated_at: new Date().toISOString(), + maintainers: rows, + }; + } +} diff --git a/packages/das/src/webhook/github-fetcher.service.ts b/packages/das/src/webhook/github-fetcher.service.ts index 88ad900..967e057 100644 --- a/packages/das/src/webhook/github-fetcher.service.ts +++ b/packages/das/src/webhook/github-fetcher.service.ts @@ -20,6 +20,11 @@ interface InstallationToken { expiresAt: number; } +interface ClosingIssueReference { + number?: number; + repository?: { nameWithOwner?: string } | null; +} + // Files larger than this are stored with null content (AST parsing is wasteful past this). const MAX_FILE_SIZE_BYTES = 1_000_000; @@ -280,7 +285,10 @@ export class GitHubFetcherService implements OnModuleInit { bodyText lastEditedAt closingIssuesReferences(first: 10) { - nodes { number } + nodes { + number + repository { nameWithOwner } + } } } } @@ -310,12 +318,29 @@ export class GitHubFetcherService implements OnModuleInit { const nodes = pr.closingIssuesReferences?.nodes ?? []; return { - closingIssueNumbers: nodes.map((n: { number: number }) => n.number), + closingIssueNumbers: this.sameRepoClosingIssueNumbers( + repoFullName, + nodes, + ), body: pr.bodyText ?? null, lastEditedAt: pr.lastEditedAt ?? null, }; } + private sameRepoClosingIssueNumbers( + repoFullName: string, + nodes: ClosingIssueReference[], + ): number[] { + const expectedRepo = repoFullName.toLowerCase(); + return nodes + .filter( + (node) => + node.repository?.nameWithOwner?.toLowerCase() === expectedRepo, + ) + .map((node) => node.number) + .filter((number): number is number => typeof number === "number"); + } + // --- PR files + contents (REST for list, batched GraphQL for contents) --- /** @@ -380,10 +405,9 @@ export class GitHubFetcherService implements OnModuleInit { // 3. Fetch file contents in batches (base + head in one GraphQL call each) if (!pr.headSha) { - this.logger.warn( - `PR ${repoFullName}#${prNumber} has no head SHA — skipping content fetch`, + throw new Error( + `PR ${repoFullName}#${prNumber} has no head SHA; cannot fetch content`, ); - return; } // Prefer merge-base SHA (true common ancestor) over base SHA for @@ -511,10 +535,9 @@ export class GitHubFetcherService implements OnModuleInit { batchSize = newSize; // Retry same i with smaller batch } else { - this.logger.warn( - `GraphQL content batch failed at min size ${minBatchSize}: ${err}. Skipping batch.`, + throw new Error( + `GraphQL content batch failed at min size ${minBatchSize}: ${err}`, ); - i += batch.length; } } } @@ -884,11 +907,14 @@ export class GitHubFetcherService implements OnModuleInit { authorAssociation labels(first: 10) { nodes { name } } timelineItems( - itemTypes: [LABELED_EVENT, UNLABELED_EVENT] + itemTypes: [LABELED_EVENT, UNLABELED_EVENT, TRANSFERRED_EVENT] first: 30 ) { nodes { __typename + ... on TransferredEvent { + createdAt + } ... on LabeledEvent { createdAt label { name } @@ -966,6 +992,15 @@ export class GitHubFetcherService implements OnModuleInit { ), }; + if ( + (issue.timelineItems?.nodes ?? []).some( + (node: { __typename?: string }) => + node.__typename === "TransferredEvent", + ) + ) { + issueData.isTransferred = true; + } + if (issue.state === "OPEN") { issueData.solvedByPr = null; } @@ -987,8 +1022,10 @@ export class GitHubFetcherService implements OnModuleInit { } /** - * Upsert a list of LABELED_EVENT / UNLABELED_EVENT timeline nodes into - * the label_events table. Actor role is resolved at read time via + * Insert LABELED_EVENT / UNLABELED_EVENT timeline nodes into label_events. + * Idempotent: relies on the uq_label_events_natural_key UNIQUE index so + * re-running backfill (or BullMQ retries) collapses to a no-op for events + * already written. Actor role is resolved at read time via * contributor_repo_roles using stored PR/issue, review, and comment * association evidence; GraphQL's actor type doesn't expose authorAssociation. */ @@ -998,23 +1035,28 @@ export class GitHubFetcherService implements OnModuleInit { targetType: "pr" | "issue", nodes: any[], ): Promise { - for (const node of nodes) { - if (!node || !node.label?.name || !node.createdAt) continue; - const action = - node.__typename === "LabeledEvent" ? "labeled" : "unlabeled"; - - await this.labelEventRepo.save({ + const rows = nodes + .filter((node) => node && node.label?.name && node.createdAt) + .map((node) => ({ repoFullName, targetNumber, targetType, labelName: node.label.name, - action, + action: node.__typename === "LabeledEvent" ? "labeled" : "unlabeled", actorGithubId: node.actor?.databaseId ? String(node.actor.databaseId) : null, actorLogin: node.actor?.login ?? null, timestamp: node.createdAt, - }); - } + })); + + if (rows.length === 0) return; + + await this.labelEventRepo + .createQueryBuilder() + .insert() + .values(rows) + .orIgnore() + .execute(); } } diff --git a/packages/das/src/webhook/handlers/comment.handler.ts b/packages/das/src/webhook/handlers/comment.handler.ts index ee104ea..d5760e7 100644 --- a/packages/das/src/webhook/handlers/comment.handler.ts +++ b/packages/das/src/webhook/handlers/comment.handler.ts @@ -2,13 +2,15 @@ import { Injectable } from "@nestjs/common"; import { InjectRepository } from "@nestjs/typeorm"; import { Repository } from "typeorm"; -import { Comment } from "../../entities"; +import { Comment, Repo } from "../../entities"; @Injectable() export class CommentHandler { constructor( @InjectRepository(Comment) private readonly commentRepo: Repository, + @InjectRepository(Repo) + private readonly repoRepo: Repository, ) {} async handle(payload: Record): Promise { @@ -20,6 +22,9 @@ export class CommentHandler { repoFullName, commentId: String(comment.id), }); + await this.repoRepo.update(repoFullName, { + lastEventAt: new Date().toISOString(), + }); return; } @@ -40,5 +45,9 @@ export class CommentHandler { }; await this.commentRepo.upsert(data, ["repoFullName", "commentId"]); + + await this.repoRepo.update(repoFullName, { + lastEventAt: new Date().toISOString(), + }); } } diff --git a/packages/das/src/webhook/handlers/issue.handler.ts b/packages/das/src/webhook/handlers/issue.handler.ts index a94edbc..ac54368 100644 --- a/packages/das/src/webhook/handlers/issue.handler.ts +++ b/packages/das/src/webhook/handlers/issue.handler.ts @@ -40,6 +40,10 @@ export class IssueHandler { data.solvedByPr = null; } + if (payload.action === "transferred") { + data.isTransferred = true; + } + // The `edited` action fires specifically for body or title changes. // Use the webhook's updated_at as the precise edit timestamp — for // other actions (labeled, closed, commented, etc.) don't touch diff --git a/packages/das/src/webhook/handlers/label.handler.ts b/packages/das/src/webhook/handlers/label.handler.ts index 5304fa1..ab813bb 100644 --- a/packages/das/src/webhook/handlers/label.handler.ts +++ b/packages/das/src/webhook/handlers/label.handler.ts @@ -34,18 +34,27 @@ export class LabelHandler { source === "pr" ? payload.pull_request.number : payload.issue.number; // Append to label_events log. Actor's repo role is resolved at read time - // via contributor_repo_roles using stored PR/issue, review, and comment - // association evidence; label actors themselves don't expose it. - await this.labelEventRepo.save({ - repoFullName, - targetNumber, - targetType: source, - labelName: label.name, - action, - actorGithubId: sender ? String(sender.id) : null, - actorLogin: sender?.login ?? null, - timestamp: new Date().toISOString(), - }); + // via contributor_repo_roles (see pr_labels_by_actor view) using stored + // PR/issue, review, and comment association evidence — neither the webhook + // sender nor GraphQL LabeledEvent.actor expose author_association. + // orIgnore() makes the insert idempotent under the uq_label_events_natural_key + // constraint; same-delivery retries are already gated upstream by + // webhook_deliveries, this is defense-in-depth. + await this.labelEventRepo + .createQueryBuilder() + .insert() + .values({ + repoFullName, + targetNumber, + targetType: source, + labelName: label.name, + action, + actorGithubId: sender ? String(sender.id) : null, + actorLogin: sender?.login ?? null, + timestamp: new Date().toISOString(), + }) + .orIgnore() + .execute(); // Update current labels snapshot on the parent row const currentLabels: string[] = diff --git a/packages/das/src/webhook/handlers/review-comment.handler.ts b/packages/das/src/webhook/handlers/review-comment.handler.ts index 3b957da..0b9b350 100644 --- a/packages/das/src/webhook/handlers/review-comment.handler.ts +++ b/packages/das/src/webhook/handlers/review-comment.handler.ts @@ -2,13 +2,15 @@ import { Injectable } from "@nestjs/common"; import { InjectRepository } from "@nestjs/typeorm"; import { Repository } from "typeorm"; -import { ReviewComment } from "../../entities"; +import { Repo, ReviewComment } from "../../entities"; @Injectable() export class ReviewCommentHandler { constructor( @InjectRepository(ReviewComment) private readonly reviewCommentRepo: Repository, + @InjectRepository(Repo) + private readonly repoRepo: Repository, ) {} async handle(payload: Record): Promise { @@ -20,6 +22,9 @@ export class ReviewCommentHandler { repoFullName, commentId: String(comment.id), }); + await this.repoRepo.update(repoFullName, { + lastEventAt: new Date().toISOString(), + }); return; } @@ -41,5 +46,9 @@ export class ReviewCommentHandler { }; await this.reviewCommentRepo.upsert(data, ["repoFullName", "commentId"]); + + await this.repoRepo.update(repoFullName, { + lastEventAt: new Date().toISOString(), + }); } } diff --git a/packages/das/src/webhook/handlers/review.handler.ts b/packages/das/src/webhook/handlers/review.handler.ts index a2f14af..3a5f573 100644 --- a/packages/das/src/webhook/handlers/review.handler.ts +++ b/packages/das/src/webhook/handlers/review.handler.ts @@ -2,13 +2,15 @@ import { Injectable } from "@nestjs/common"; import { InjectRepository } from "@nestjs/typeorm"; import { Repository } from "typeorm"; -import { Review } from "../../entities"; +import { Repo, Review } from "../../entities"; @Injectable() export class ReviewHandler { constructor( @InjectRepository(Review) private readonly reviewRepo: Repository, + @InjectRepository(Repo) + private readonly repoRepo: Repository, ) {} async handle(payload: Record): Promise { @@ -34,5 +36,9 @@ export class ReviewHandler { "reviewerGithubId", "submittedAt", ]); + + await this.repoRepo.update(repoFullName, { + lastEventAt: new Date().toISOString(), + }); } } diff --git a/packages/db/07_label_events.sql b/packages/db/07_label_events.sql index 69c3730..8c2374f 100644 --- a/packages/db/07_label_events.sql +++ b/packages/db/07_label_events.sql @@ -16,3 +16,8 @@ CREATE TABLE IF NOT EXISTS label_events ( ); CREATE INDEX IF NOT EXISTS idx_label_events_target ON label_events(repo_full_name, target_number, timestamp); + +CREATE UNIQUE INDEX IF NOT EXISTS uq_label_events_natural_key + ON label_events (repo_full_name, target_number, target_type, + label_name, action, timestamp) + NULLS NOT DISTINCT;