Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 13 additions & 1 deletion packages/das/src/api/api.module.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13,10 +13,14 @@ import { FETCH_QUEUE } from "../queue/constants";
import { AdminController } from "./admin.controller";
import { RequireApiKeyGuard } from "./require-api-key.guard";
import { HealthController } from "./health.controller";
import { DashboardController } from "./dashboard/dashboard.controller";
import { DashboardService } from "./dashboard/dashboard.service";
import { MinersController } from "./miners/miners.controller";
import { MinersService } from "./miners/miners.service";
import { PullsController } from "./pulls/pulls.controller";
import { PullsService } from "./pulls/pulls.service";
import { ReposController } from "./repos/repos.controller";
import { ReposService } from "./repos/repos.service";

@Module({
imports: [
Expand All @@ -31,11 +35,19 @@ import { PullsService } from "./pulls/pulls.service";
BullModule.registerQueue({ name: FETCH_QUEUE }),
],
controllers: [
DashboardController,
MinersController,
PullsController,
ReposController,
AdminController,
HealthController,
],
providers: [MinersService, PullsService, RequireApiKeyGuard],
providers: [
DashboardService,
MinersService,
PullsService,
ReposService,
RequireApiKeyGuard,
],
})
export class ApiModule {}
37 changes: 37 additions & 0 deletions packages/das/src/api/dashboard/dashboard.controller.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
import { BadRequestException, Controller, Get, Query } from "@nestjs/common";
import { ApiOperation, ApiQuery, ApiTags } from "@nestjs/swagger";
import { DashboardService } from "./dashboard.service";

@ApiTags("Dashboard")
@Controller("api/v1/dashboard")
export class DashboardController {
constructor(private readonly dashboard: DashboardService) {}

@Get("issues")
@ApiOperation({
summary: "Slim issue rows for dashboard trend aggregation",
description:
"Returns every issue with `created_at` on or after `since`, plus " +
"every CLOSED issue whose `closed_at` is on or after `since`. " +
"The mirror is intentionally roster-blind: every issue is returned " +
"regardless of author. The dashboard blends with the gittensor API " +
"miner roster client-side to filter to subnet authors. " +
"Designed as a single bulk replacement for the dashboard's per-miner " +
"fan-out against `/miners/<id>/issues` (one call instead of N).",
})
@ApiQuery({
name: "since",
required: true,
description: "ISO timestamp — earliest creation/close date to include.",
})
async getIssues(@Query("since") since?: string): Promise<unknown> {
if (!since) {
throw new BadRequestException("`since` query parameter is required");
}
const parsed = new Date(since);
if (Number.isNaN(parsed.getTime())) {
throw new BadRequestException("`since` must be a valid ISO timestamp");
}
return this.dashboard.getIssues(parsed.toISOString());
}
}
58 changes: 58 additions & 0 deletions packages/das/src/api/dashboard/dashboard.service.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
/* eslint-disable @typescript-eslint/no-unsafe-return, @typescript-eslint/no-unsafe-assignment */
import { Injectable } from "@nestjs/common";
import { DataSource } from "typeorm";

interface DashboardIssueRow {
repo_full_name: string;
issue_number: number;
author_github_id: string | null;
created_at: string;
closed_at: string | null;
state: string;
state_reason: string | null;
solving_pr: { merged_at: string } | null;
}

@Injectable()
export class DashboardService {
constructor(private readonly dataSource: DataSource) {}

async getIssues(since: string): Promise<{
since: string;
generated_at: string;
issues: DashboardIssueRow[];
}> {
const rows = await this.dataSource.query(
`
SELECT
LOWER(i.repo_full_name) AS repo_full_name,
i.issue_number,
i.author_github_id,
i.created_at,
i.closed_at,
i.state,
i.state_reason,
(
SELECT json_build_object('merged_at', sp.merged_at)
FROM pull_requests sp
WHERE sp.repo_full_name = i.repo_full_name
AND sp.pr_number = i.solved_by_pr
AND sp.merged_at IS NOT NULL
LIMIT 1
) AS solving_pr
FROM issues i
WHERE
i.created_at >= $1
OR (i.state = 'CLOSED' AND i.closed_at >= $1)
ORDER BY i.created_at DESC
`,
[since],
);

return {
since,
generated_at: new Date().toISOString(),
issues: rows as DashboardIssueRow[],
};
}
}
27 changes: 27 additions & 0 deletions packages/das/src/api/repos/repos.controller.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
import { Controller, Get, Param } from "@nestjs/common";
import { ApiOperation, ApiParam, ApiTags } from "@nestjs/swagger";
import { ReposService } from "./repos.service";

@ApiTags("Repos")
@Controller("api/v1/repos")
export class ReposController {
constructor(private readonly repos: ReposService) {}

@Get(":owner/:repo/maintainers")
@ApiOperation({
summary: "Maintainer-role contributors for a repo",
description:
"Returns users whose latest known GitHub association for the repo " +
"is OWNER, MEMBER, or COLLABORATOR, synthesized from PR/issue/" +
"review/comment activity (contributor_repo_roles view). An unknown " +
"repo returns an empty maintainers list, not a 404.",
})
@ApiParam({ name: "owner", description: "Repository owner (org or user)" })
@ApiParam({ name: "repo", description: "Repository name" })
async getMaintainers(
@Param("owner") owner: string,
@Param("repo") repo: string,
): Promise<unknown> {
return this.repos.getMaintainers(owner, repo);
}
}
41 changes: 41 additions & 0 deletions packages/das/src/api/repos/repos.service.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
/* eslint-disable @typescript-eslint/no-unsafe-return, @typescript-eslint/no-unsafe-assignment */
import { Injectable } from "@nestjs/common";
import { DataSource } from "typeorm";

@Injectable()
export class ReposService {
constructor(private readonly dataSource: DataSource) {}

async getMaintainers(
owner: string,
repo: string,
): Promise<{
repo_full_name: string;
generated_at: string;
maintainers: unknown[];
}> {
const repoFullName = `${owner}/${repo}`;

// The association literals must stay in sync with gittensor
// constants.py MAINTAINER_ASSOCIATIONS.
const rows = await this.dataSource.query(
`
SELECT
cr.author_github_id AS github_id,
cr.author_login AS login,
cr.author_association AS association
FROM contributor_repo_roles cr
WHERE LOWER(cr.repo_full_name) = LOWER($1)
AND cr.author_association IN ('OWNER', 'MEMBER', 'COLLABORATOR')
ORDER BY cr.author_github_id
`,
[repoFullName],
);

return {
repo_full_name: repoFullName.toLowerCase(),
generated_at: new Date().toISOString(),
maintainers: rows,
};
}
}
82 changes: 62 additions & 20 deletions packages/das/src/webhook/github-fetcher.service.ts
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,11 @@ interface InstallationToken {
expiresAt: number;
}

interface ClosingIssueReference {
number?: number;
repository?: { nameWithOwner?: string } | null;
}

// Files larger than this are stored with null content (AST parsing is wasteful past this).
const MAX_FILE_SIZE_BYTES = 1_000_000;

Expand Down Expand Up @@ -280,7 +285,10 @@ export class GitHubFetcherService implements OnModuleInit {
bodyText
lastEditedAt
closingIssuesReferences(first: 10) {
nodes { number }
nodes {
number
repository { nameWithOwner }
}
}
}
}
Expand Down Expand Up @@ -310,12 +318,29 @@ export class GitHubFetcherService implements OnModuleInit {
const nodes = pr.closingIssuesReferences?.nodes ?? [];

return {
closingIssueNumbers: nodes.map((n: { number: number }) => n.number),
closingIssueNumbers: this.sameRepoClosingIssueNumbers(
repoFullName,
nodes,
),
body: pr.bodyText ?? null,
lastEditedAt: pr.lastEditedAt ?? null,
};
}

private sameRepoClosingIssueNumbers(
repoFullName: string,
nodes: ClosingIssueReference[],
): number[] {
const expectedRepo = repoFullName.toLowerCase();
return nodes
.filter(
(node) =>
node.repository?.nameWithOwner?.toLowerCase() === expectedRepo,
)
.map((node) => node.number)
.filter((number): number is number => typeof number === "number");
}

// --- PR files + contents (REST for list, batched GraphQL for contents) ---

/**
Expand Down Expand Up @@ -380,10 +405,9 @@ export class GitHubFetcherService implements OnModuleInit {

// 3. Fetch file contents in batches (base + head in one GraphQL call each)
if (!pr.headSha) {
this.logger.warn(
`PR ${repoFullName}#${prNumber} has no head SHA — skipping content fetch`,
throw new Error(
`PR ${repoFullName}#${prNumber} has no head SHA; cannot fetch content`,
);
return;
}

// Prefer merge-base SHA (true common ancestor) over base SHA for
Expand Down Expand Up @@ -511,10 +535,9 @@ export class GitHubFetcherService implements OnModuleInit {
batchSize = newSize;
// Retry same i with smaller batch
} else {
this.logger.warn(
`GraphQL content batch failed at min size ${minBatchSize}: ${err}. Skipping batch.`,
throw new Error(
`GraphQL content batch failed at min size ${minBatchSize}: ${err}`,
);
i += batch.length;
}
}
}
Expand Down Expand Up @@ -884,11 +907,14 @@ export class GitHubFetcherService implements OnModuleInit {
authorAssociation
labels(first: 10) { nodes { name } }
timelineItems(
itemTypes: [LABELED_EVENT, UNLABELED_EVENT]
itemTypes: [LABELED_EVENT, UNLABELED_EVENT, TRANSFERRED_EVENT]
first: 30
) {
nodes {
__typename
... on TransferredEvent {
createdAt
}
... on LabeledEvent {
createdAt
label { name }
Expand Down Expand Up @@ -966,6 +992,15 @@ export class GitHubFetcherService implements OnModuleInit {
),
};

if (
(issue.timelineItems?.nodes ?? []).some(
(node: { __typename?: string }) =>
node.__typename === "TransferredEvent",
)
) {
issueData.isTransferred = true;
}

if (issue.state === "OPEN") {
issueData.solvedByPr = null;
}
Expand All @@ -987,8 +1022,10 @@ export class GitHubFetcherService implements OnModuleInit {
}

/**
* Upsert a list of LABELED_EVENT / UNLABELED_EVENT timeline nodes into
* the label_events table. Actor role is resolved at read time via
* Insert LABELED_EVENT / UNLABELED_EVENT timeline nodes into label_events.
* Idempotent: relies on the uq_label_events_natural_key UNIQUE index so
* re-running backfill (or BullMQ retries) collapses to a no-op for events
* already written. Actor role is resolved at read time via
* contributor_repo_roles using stored PR/issue, review, and comment
* association evidence; GraphQL's actor type doesn't expose authorAssociation.
*/
Expand All @@ -998,23 +1035,28 @@ export class GitHubFetcherService implements OnModuleInit {
targetType: "pr" | "issue",
nodes: any[],
): Promise<void> {
for (const node of nodes) {
if (!node || !node.label?.name || !node.createdAt) continue;
const action =
node.__typename === "LabeledEvent" ? "labeled" : "unlabeled";

await this.labelEventRepo.save({
const rows = nodes
.filter((node) => node && node.label?.name && node.createdAt)
.map((node) => ({
repoFullName,
targetNumber,
targetType,
labelName: node.label.name,
action,
action: node.__typename === "LabeledEvent" ? "labeled" : "unlabeled",
actorGithubId: node.actor?.databaseId
? String(node.actor.databaseId)
: null,
actorLogin: node.actor?.login ?? null,
timestamp: node.createdAt,
});
}
}));

if (rows.length === 0) return;

await this.labelEventRepo
.createQueryBuilder()
.insert()
.values(rows)
.orIgnore()
.execute();
}
}
Loading