From 73e6a0567db5a80f92e6e91ea7b8eb7cf80dcbbe Mon Sep 17 00:00:00 2001 From: anderdc Date: Tue, 28 Apr 2026 15:25:43 -0500 Subject: [PATCH 01/15] Add open source templates and contributor docs --- .github/CODE_OF_CONDUCT.md | 68 ++++++++++++++ .github/ISSUE_TEMPLATE/bug_report.md | 33 +++++++ .github/ISSUE_TEMPLATE/config.yml | 5 + .github/ISSUE_TEMPLATE/feature_request.md | 17 ++++ .github/PULL_REQUEST_TEMPLATE.md | 27 ++++++ CONTRIBUTING.md | 109 ++++++++++++++++++++++ LICENSE | 21 +++++ SECURITY.md | 40 ++++++++ 8 files changed, 320 insertions(+) create mode 100644 .github/CODE_OF_CONDUCT.md create mode 100644 .github/ISSUE_TEMPLATE/bug_report.md create mode 100644 .github/ISSUE_TEMPLATE/config.yml create mode 100644 .github/ISSUE_TEMPLATE/feature_request.md create mode 100644 .github/PULL_REQUEST_TEMPLATE.md create mode 100644 CONTRIBUTING.md create mode 100644 LICENSE create mode 100644 SECURITY.md diff --git a/.github/CODE_OF_CONDUCT.md b/.github/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..3a54b26 --- /dev/null +++ b/.github/CODE_OF_CONDUCT.md @@ -0,0 +1,68 @@ +## Pledge + +We pledge to make our community welcoming, safe, and equitable for all. + +We are committed to fostering an environment that respects and promotes the dignity, rights, and contributions of all individuals, regardless of characteristics including race, ethnicity, caste, color, age, physical characteristics, neurodiversity, disability, sex or gender, gender identity or expression, sexual orientation, language, philosophy or religion, national or social origin, socio-economic position, level of education, or other status. The same privileges of participation are extended to everyone who participates in good faith and in accordance with this Covenant. + +### Encouraged Behaviors + +While acknowledging differences in social norms, we all strive to meet our community's expectations for positive behavior. We also understand that our words and actions may be interpreted differently than we intend based on culture, background, or native language. + +With these considerations in mind, we agree to behave mindfully toward each other and act in ways that center our shared values, including: + +- Respecting the purpose of our community, our activities, and our ways of gathering. +- Engaging kindly and honestly with others. +- Respecting different viewpoints and experiences. +- Taking responsibility for our actions and contributions. +- Gracefully giving and accepting constructive feedback. +- Committing to repairing harm when it occurs. +- Behaving in other ways that promote and sustain the well-being of our community. + +## Restricted Behaviors + +We agree to restrict the following behaviors in our community. Instances, threats, and promotion of these behaviors are violations of this Code of Conduct. + +- Harassment. Violating explicitly expressed boundaries or engaging in unnecessary personal attention after any clear request to stop. +- Character attacks. Making insulting, demeaning, or pejorative comments directed at a community member or group of people. +- Stereotyping or discrimination. Characterizing anyone's personality or behavior on the basis of immutable identities or traits. +- Sexualization. Behaving in a way that would generally be considered inappropriately intimate in the context or purpose of the community. +- Violating confidentiality. Sharing or acting on someone's personal or private information without their permission. +- Endangerment. Causing, encouraging, or threatening violence or other harm toward any person or group. +- Behaving in other ways that threaten the well-being of our community. + +### Other Restrictions + +- Misleading identity. Impersonating someone else for any reason, or pretending to be someone else to evade enforcement actions. +- Failing to credit sources. Not properly crediting the sources of content you contribute. +- Irresponsible communication. Failing to responsibly present content which includes, links or describes any other restricted behaviors. + +## Reporting an Issue + +Tensions can occur between community members even when they are trying their best to collaborate. Not every conflict represents a code of conduct violation, and this Code of Conduct reinforces encouraged behaviors and norms that can help avoid conflicts and minimize harm. + +When an incident does occur, it is important to report it promptly. To report a possible violation, contact the project maintainers directly through GitHub issues or discussions. + +Community Moderators take reports of violations seriously and will investigate all reports of code of conduct violations. In order to honor the values of safety and confidentiality, enforcement actions are carried out as transparently as possible while prioritizing the safety and privacy of those involved. + +### Addressing and Repairing Harm + +If an investigation by the Community Moderators finds that this Code of Conduct has been violated, the following enforcement ladder may be used to determine how best to repair harm, based on the incident's impact on the individuals involved and the community as a whole. Depending on the severity of a violation, lower rungs on the ladder may be skipped. + +- **Warning** + - Event: A violation involving a single incident or series of incidents. + - Consequence: A private, written warning from the Community Moderators. + - Repair: Examples of repair include a private written apology, acknowledgement of responsibility, and seeking clarification on expectations. +- **Temporary Suspension** + - Event: A pattern of repeated violation which the Community Moderators have tried to address with warnings, or a single serious violation. + - Consequence: A private written warning with conditions for return from suspension. + - Repair: Examples of repair include respecting the spirit of the suspension, meeting the specified conditions for return, and being thoughtful about how to reintegrate with the community when the suspension is lifted. +- **Permanent Ban** + - Event: A pattern of repeated code of conduct violations that other steps on the ladder have failed to resolve, or a violation so serious that the Community Moderators determine there is no way to keep the community safe with this person as a member. + - Consequence: Access to all community spaces, tools, and communication channels is removed. + - Repair: There is no possible repair in cases of this severity. + +This enforcement ladder is intended as a guideline. It does not limit the ability of Community Managers to use their discretion and judgment, in keeping with the best interests of our community. + +## Scope + +This Code of Conduct applies within all community spaces, and also applies when an individual is officially representing the community in public or other spaces. Examples of representing our community include using an official email address, posting via an official social media account, or acting as an appointed representative at an online or offline event. diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 0000000..cffc411 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,33 @@ +--- +name: Bug Report +about: Report a bug or unexpected behavior +labels: bug +--- + +## Description + + + +## Steps to Reproduce + +1. +2. +3. + +## Expected Behavior + + + +## Actual Behavior + + + +## Environment + +- OS: +- Runtime/Node version: +- Browser (if applicable): + +## Additional Context + + diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 0000000..da939ec --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,5 @@ +blank_issues_enabled: true +contact_links: + - name: Security Vulnerability + url: https://github.com/entrius/das-github-mirror/security/advisories/new + about: Report security vulnerabilities privately via GitHub Security Advisories diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 0000000..47a71e0 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,17 @@ +--- +name: Feature Request +about: Suggest a new feature or improvement +labels: enhancement +--- + +## Summary + + + +## Motivation + + + +## Proposed Solution + + diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 0000000..78b710f --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,27 @@ +## Summary + + + +## Related Issues + + + +## Type of Change + +- [ ] Bug fix +- [ ] New feature +- [ ] Refactor +- [ ] Documentation +- [ ] Other (describe below) + +## Testing + + + +## Checklist + +- [ ] I have read the [Contributing Guide](./CONTRIBUTING.md) +- [ ] Code builds without errors +- [ ] New and existing tests pass (if applicable) +- [ ] Documentation updated (if applicable) +- [ ] No unnecessary dependencies added diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..48cf627 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,109 @@ +# das-github-mirror Contributor Guide + +## Getting Started + +Before contributing, please: + +1. Read the [README](./README.md) to understand the project +2. Familiarize yourself with the project structure and tech stack +3. Check existing issues and PRs to avoid duplicate work + +## Local Development + +1. Ensure you have the required runtime installed (check README for version requirements) +2. Clone the repo and install dependencies +3. Copy `.env.example` to `.env` and configure as needed +4. Follow the README to start the development server + +## Creating Issues + +When opening an issue, use the appropriate template: + +- **[Bug Report](.github/ISSUE_TEMPLATE/bug_report.md)** - Report bugs or unexpected behavior. Include steps to reproduce, expected vs actual behavior, and environment details. +- **[Feature Request](.github/ISSUE_TEMPLATE/feature_request.md)** - Suggest new features or improvements. Explain the motivation and proposed solution. +- **Blank Issue** - For issues that don't fit the above templates. + +For security vulnerabilities, **do not create a public issue**. Report them privately via [GitHub Security Advisories](https://github.com/entrius/das-github-mirror/security/advisories/new). + +## Pull Request Process + +### 1. Create Your Branch + +- Branch off of `test` and target it with your PR. PRs that target the wrong base branch will be closed without review. +- Ensure there are no conflicts before submitting + +### 2. Make Your Changes + +- Write clean, well-documented code +- Follow existing code patterns and architecture +- Update documentation if applicable +- Ensure everything builds and runs correctly before submitting + +### 3. Submit Pull Request + +1. Push your branch to the repository +2. Open a PR targeting `test` +3. Fill out the [PR template](.github/PULL_REQUEST_TEMPLATE.md): + - **Summary**: Clear description of changes + - **Related Issues**: Link issues using `Fixes #123` or `Closes #456` + - **Type of Change**: Select bug fix, new feature, refactor, documentation, or other + - **Testing**: Confirm manual testing performed + - **Checklist**: Verify your changes meet the repo's standards + +### 4. Code Review + +- Reviewers will be assigned automatically +- Address review comments promptly + +### Issue Scope + +PRs should focus on the linked issue. Minor incidental changes are fine. PRs dominated by unrelated changes (>50% of the diff) will be asked to scope down. + +### PR Iteration Expectations + +The repository runs an automated maintainer agent that may close PRs in the following cases: + +- Failing CI for 12+ hours with no fix pushed +- Unresolved merge conflicts for 12+ hours with no resolution push +- Requested changes from a maintainer for 12+ hours with no follow-up commits + +## PR Labels + +Apply appropriate labels to help categorize and track your contribution: + +- `bug` - Bug fixes +- `feature` - New feature additions +- `enhancement` - Improvements to existing features +- `refactor` - Code refactoring without functionality changes +- `documentation` - Documentation updates + +## Code Standards + +### Quality Expectations + +- Follow repository conventions (commenting style, variable naming, etc.) +- Use sensible component decomposition to keep files manageable +- Write clean, readable, maintainable code +- Avoid modifying unrelated files +- Avoid adding unnecessary dependencies +- Ensure all build checks pass before submitting + +## Branches + +### `test` + +**Purpose**: Main development and production-ready code + +**Restrictions**: + +- Requires pull request +- Requires all checks to pass +- Requires at least one approval + +## License + +By contributing to das-github-mirror, you agree that your contributions will be licensed under the project's MIT license. + +--- + +Thank you for contributing to das-github-mirror! diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..72f6ea3 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 entrius + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..b4ca1db --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,40 @@ +# Security Policy + +## Reporting a Vulnerability + +If you discover a security vulnerability in das-github-mirror or any of its components, we strongly encourage you to report it responsibly. + +Please **do not publicly disclose** the vulnerability until we have had a reasonable chance to address it. + +### Confidential Reporting + +To report a vulnerability, you can use any of the following methods: + +- Use [GitHub Security Advisories](https://github.com/entrius/das-github-mirror/security/advisories/new) to report privately. + +### What to Include + +When reporting a vulnerability, please provide as much detail as possible: + +- Affected component/files +- Version or commit hash +- Description of the vulnerability +- Steps to reproduce (if possible) +- Impact assessment +- Any potential mitigations or recommendations + +--- + +## Response Process + +1. We will acknowledge your report within **48 hours**. +2. We will investigate and confirm the issue. +3. If confirmed, we will coordinate on a fix and set an embargo period if needed. +4. A fix will be developed, tested, and released as soon as possible. +5. You will be credited (if you wish) in the security section of our release notes. + +--- + +## Thank You + +We appreciate your efforts in keeping das-github-mirror secure and responsible. From 433d2b1ed6ed0049bbf078f7780028c8e24d10aa Mon Sep 17 00:00:00 2001 From: Daniil Sivak Date: Sun, 10 May 2026 23:54:14 +0300 Subject: [PATCH 02/15] Normalize response shapes (#8) --- packages/das/src/api/miners/miners.service.ts | 30 +++++++++++-------- packages/das/src/api/pulls/pulls.service.ts | 11 +++++-- .../webhook/handlers/pull-request.handler.ts | 2 +- packages/db/02_pull_requests.sql | 4 ++- 4 files changed, 30 insertions(+), 17 deletions(-) diff --git a/packages/das/src/api/miners/miners.service.ts b/packages/das/src/api/miners/miners.service.ts index bb93bc0..e4f1619 100644 --- a/packages/das/src/api/miners/miners.service.ts +++ b/packages/das/src/api/miners/miners.service.ts @@ -20,13 +20,13 @@ export class MinersService { const rows = await this.dataSource.query( ` SELECT - p.repo_full_name, + LOWER(p.repo_full_name) AS repo_full_name, p.pr_number, - p.title, + COALESCE(p.title, '') AS title, p.body, p.state, p.author_github_id, - p.author_login, + COALESCE(p.author_login, '') AS author_login, p.author_association, p.created_at, p.closed_at, @@ -35,14 +35,14 @@ export class MinersService { p.merged_by_login, p.base_ref, p.head_ref, - p.head_repo_full_name, + LOWER(p.head_repo_full_name) AS head_repo_full_name, r.default_branch, p.head_sha, p.base_sha, p.merge_base_sha, - p.additions, - p.deletions, - p.commits_count, + COALESCE(p.additions, 0) AS additions, + COALESCE(p.deletions, 0) AS deletions, + COALESCE(p.commits_count, 0) AS commits_count, p.scoring_data_stored, (p.last_edited_at IS NOT NULL AND p.merged_at IS NOT NULL AND p.last_edited_at > p.merged_at) AS edited_after_merge, @@ -72,7 +72,7 @@ export class MinersService { COALESCE(( SELECT json_agg(json_build_object( 'number', li.issue_number, - 'title', li.issue_title, + 'title', COALESCE(li.issue_title, ''), 'state', li.issue_state, 'state_reason', li.issue_state_reason, 'author_github_id', li.issue_author_github_id, @@ -134,9 +134,9 @@ export class MinersService { const rows = await this.dataSource.query( ` SELECT - i.repo_full_name, + LOWER(i.repo_full_name) AS repo_full_name, i.issue_number, - i.title, + COALESCE(i.title, '') AS title, i.state, i.state_reason, i.author_github_id, @@ -188,8 +188,10 @@ export class MinersService { AND plt.pr_number = sp.pr_number ), '[]'::json), 'review_summary', json_build_object( - 'maintainer_changes_requested_count', - COALESCE(rs.maintainer_changes_requested_count, 0) + 'maintainer_changes_requested_count', COALESCE(rs.maintainer_changes_requested_count, 0), + 'changes_requested_count', COALESCE(rs.changes_requested_count, 0), + 'approved_count', COALESCE(rs.approved_count, 0), + 'commented_count', COALESCE(rs.commented_count, 0) ) ) FROM pull_requests sp @@ -198,6 +200,10 @@ export class MinersService { AND rs.pr_number = sp.pr_number WHERE sp.repo_full_name = i.repo_full_name AND sp.pr_number = i.solved_by_pr + -- Skip null-author solving PRs (no one to credit) + AND sp.author_github_id IS NOT NULL + -- Skip corrupted MERGED-without-merged_at shape + AND NOT (sp.state = 'MERGED' AND sp.merged_at IS NULL) ) AS solving_pr FROM issues i WHERE i.author_github_id = $1 diff --git a/packages/das/src/api/pulls/pulls.service.ts b/packages/das/src/api/pulls/pulls.service.ts index 42a4d8c..f6084e3 100644 --- a/packages/das/src/api/pulls/pulls.service.ts +++ b/packages/das/src/api/pulls/pulls.service.ts @@ -16,7 +16,7 @@ export class PullsService { const rows = await this.dataSource.query( ` SELECT - p.repo_full_name, + LOWER(p.repo_full_name) AS repo_full_name, p.pr_number, p.head_sha, p.base_sha, @@ -44,8 +44,13 @@ export class PullsService { AND f.pr_number = p.pr_number ), '[]'::json) AS files FROM pull_requests p - WHERE p.repo_full_name = $1 - AND p.pr_number = $2 + -- Look up the canonical-case repo_full_name via the small repos + -- table so the pull_requests PK seek stays index-driven + WHERE p.repo_full_name = ( + SELECT repo_full_name FROM repos + WHERE LOWER(repo_full_name) = LOWER($1) + ) + AND p.pr_number = $2 `, [repoFullName, prNumber], ); diff --git a/packages/das/src/webhook/handlers/pull-request.handler.ts b/packages/das/src/webhook/handlers/pull-request.handler.ts index dd54838..3406a83 100644 --- a/packages/das/src/webhook/handlers/pull-request.handler.ts +++ b/packages/das/src/webhook/handlers/pull-request.handler.ts @@ -33,7 +33,7 @@ export class PullRequestHandler { authorLogin: pr.user.login, authorAssociation: pr.author_association, title: pr.title, - state: pr.merged ? "MERGED" : pr.state.toUpperCase(), + state: pr.merged && pr.merged_at ? "MERGED" : pr.state.toUpperCase(), createdAt: pr.created_at, closedAt: pr.closed_at ?? null, mergedAt: pr.merged_at ?? null, diff --git a/packages/db/02_pull_requests.sql b/packages/db/02_pull_requests.sql index abbf844..080a41f 100644 --- a/packages/db/02_pull_requests.sql +++ b/packages/db/02_pull_requests.sql @@ -27,7 +27,9 @@ CREATE TABLE IF NOT EXISTS pull_requests ( closing_issue_numbers INTEGER[], scoring_data_stored BOOLEAN NOT NULL DEFAULT FALSE, - PRIMARY KEY (repo_full_name, pr_number) + PRIMARY KEY (repo_full_name, pr_number), + CONSTRAINT pull_requests_merged_has_merged_at + CHECK (state != 'MERGED' OR merged_at IS NOT NULL) ); CREATE INDEX IF NOT EXISTS idx_pull_requests_author ON pull_requests(author_github_id); From 9dc9c767ab79d5ab4eb8ea4332fdb6e1560eac68 Mon Sep 17 00:00:00 2001 From: bitloi <89318445+bitloi@users.noreply.github.com> Date: Sun, 10 May 2026 19:30:45 -0300 Subject: [PATCH 03/15] fix: clear solved_by_pr when issues reopen (#24) --- .../das/src/webhook/github-fetcher.service.ts | 43 ++++++++++--------- .../das/src/webhook/handlers/issue.handler.ts | 7 ++- 2 files changed, 29 insertions(+), 21 deletions(-) diff --git a/packages/das/src/webhook/github-fetcher.service.ts b/packages/das/src/webhook/github-fetcher.service.ts index 98c8de4..d58e18f 100644 --- a/packages/das/src/webhook/github-fetcher.service.ts +++ b/packages/das/src/webhook/github-fetcher.service.ts @@ -931,26 +931,29 @@ export class GitHubFetcherService implements OnModuleInit { break; } - await this.issueRepo.upsert( - { - repoFullName, - issueNumber: issue.number, - authorGithubId: String(issue.author?.databaseId ?? ""), - authorLogin: issue.author?.login ?? null, - authorAssociation: issue.authorAssociation ?? null, - title: issue.title, - state: issue.state, // OPEN / CLOSED - stateReason: issue.stateReason ?? null, - createdAt: issue.createdAt, - closedAt: issue.closedAt ?? null, - updatedAt: issue.updatedAt ?? null, - lastEditedAt: issue.lastEditedAt ?? null, - labels: (issue.labels?.nodes ?? []).map( - (l: { name: string }) => l.name, - ), - }, - ["repoFullName", "issueNumber"], - ); + const issueData: Partial = { + repoFullName, + issueNumber: issue.number, + authorGithubId: String(issue.author?.databaseId ?? ""), + authorLogin: issue.author?.login ?? null, + authorAssociation: issue.authorAssociation ?? null, + title: issue.title, + state: issue.state, // OPEN / CLOSED + stateReason: issue.stateReason ?? null, + createdAt: issue.createdAt, + closedAt: issue.closedAt ?? null, + updatedAt: issue.updatedAt ?? null, + lastEditedAt: issue.lastEditedAt ?? null, + labels: (issue.labels?.nodes ?? []).map( + (l: { name: string }) => l.name, + ), + }; + + if (issue.state === "OPEN") { + issueData.solvedByPr = null; + } + + await this.issueRepo.upsert(issueData, ["repoFullName", "issueNumber"]); // Upsert label events for this issue await this.saveLabelTimelineEvents( diff --git a/packages/das/src/webhook/handlers/issue.handler.ts b/packages/das/src/webhook/handlers/issue.handler.ts index 98f9090..a94edbc 100644 --- a/packages/das/src/webhook/handlers/issue.handler.ts +++ b/packages/das/src/webhook/handlers/issue.handler.ts @@ -20,6 +20,7 @@ export class IssueHandler { // Skip pull request events delivered as issue events if (issue.pull_request) return; + const issueState = issue.state.toUpperCase(); const data: Partial = { repoFullName, issueNumber: issue.number, @@ -27,7 +28,7 @@ export class IssueHandler { authorLogin: issue.user.login, authorAssociation: issue.author_association, title: issue.title ?? null, - state: issue.state.toUpperCase(), + state: issueState, stateReason: issue.state_reason?.toUpperCase() ?? null, createdAt: issue.created_at, closedAt: issue.closed_at ?? null, @@ -35,6 +36,10 @@ export class IssueHandler { labels: (issue.labels ?? []).map((l: any) => l.name), }; + if (issueState === "OPEN") { + data.solvedByPr = null; + } + // The `edited` action fires specifically for body or title changes. // Use the webhook's updated_at as the precise edit timestamp — for // other actions (labeled, closed, commented, etc.) don't touch From 56cb4179c94b8aa322bbe798cbb06992d69cfe7b Mon Sep 17 00:00:00 2001 From: Ander <61125407+anderdc@users.noreply.github.com> Date: Mon, 11 May 2026 10:17:51 -0500 Subject: [PATCH 04/15] feat(miners): unbound OPEN issues when since is omitted (#27) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When `since` is provided, behavior is unchanged: OPEN created on/after, CLOSED closed on/after — the scoring window. When `since` is omitted, the response now returns all currently-OPEN issues with no time bound and no CLOSED history. Callers asking for "current open-issue load" no longer need a synthetic epoch-since workaround that pulls a full all-time payload. Lets the validator's open-issue spam-multiplier count old still-open issues that fall outside the scoring lookback window — the gap called out in entrius/gittensor#929 / PR #930. Co-authored-by: anderdc --- packages/das/src/api/miners/miners.controller.ts | 14 +++++++++----- packages/das/src/api/miners/miners.service.ts | 6 +++--- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/packages/das/src/api/miners/miners.controller.ts b/packages/das/src/api/miners/miners.controller.ts index befa9f1..61288eb 100644 --- a/packages/das/src/api/miners/miners.controller.ts +++ b/packages/das/src/api/miners/miners.controller.ts @@ -37,21 +37,25 @@ export class MinersController { @ApiOperation({ summary: "Issues authored by a miner", description: - "Returns every issue the miner has authored since the given date, " + - "including current labels with actor attribution and the PR number " + - "(if any) that solved the issue.", + "Returns issues the miner has authored, with current labels (actor " + + "attribution) and any solving PR. When `since` is provided, returns " + + "OPEN issues created on/after that date plus CLOSED issues closed " + + "on/after that date (scoring window). When `since` is omitted, " + + "returns all currently-OPEN issues with no time bound and no CLOSED " + + "history (open-issue load counting).", }) @ApiParam({ name: "githubId", description: "GitHub user ID (numeric)" }) @ApiQuery({ name: "since", required: false, description: - "ISO timestamp. Defaults to 35 days ago (midnight UTC) if omitted.", + "ISO timestamp. When omitted, the response contains all currently-" + + "OPEN issues with no time bound and no CLOSED history.", }) async getIssues( @Param("githubId") githubId: string, @Query("since") since?: string, ): Promise { - return this.miners.getIssues(githubId, MinersService.resolveSince(since)); + return this.miners.getIssues(githubId, since ?? null); } } diff --git a/packages/das/src/api/miners/miners.service.ts b/packages/das/src/api/miners/miners.service.ts index e4f1619..180953f 100644 --- a/packages/das/src/api/miners/miners.service.ts +++ b/packages/das/src/api/miners/miners.service.ts @@ -124,10 +124,10 @@ export class MinersService { async getIssues( githubId: string, - since: string, + since: string | null, ): Promise<{ github_id: string; - since: string; + since: string | null; generated_at: string; issues: unknown[]; }> { @@ -208,7 +208,7 @@ export class MinersService { FROM issues i WHERE i.author_github_id = $1 AND ( - (i.state = 'OPEN' AND i.created_at >= $2) + (i.state = 'OPEN' AND ($2::timestamptz IS NULL OR i.created_at >= $2)) OR (i.state = 'CLOSED' AND i.closed_at >= $2) ) ORDER BY i.created_at DESC From a4e4c115b7b0fe36650422f926af6b3c5c010fe3 Mon Sep 17 00:00:00 2001 From: MkDev11 <94194147+MkDev11@users.noreply.github.com> Date: Mon, 11 May 2026 15:57:06 -0700 Subject: [PATCH 05/15] Fix stale PR file fetch completion (#12) * Fix stale PR file fetch completion * Fix stale PR file write race * Run tests before DAS build * Drop contributor test artifacts * Slim stale PR file completion fix --------- Co-authored-by: mkdev11 --- packages/das/src/queue/constants.ts | 11 ++ packages/das/src/queue/fetch.processor.ts | 112 ++++++++++++++---- .../das/src/webhook/github-fetcher.service.ts | 16 ++- .../webhook/handlers/pull-request.handler.ts | 13 +- 4 files changed, 125 insertions(+), 27 deletions(-) diff --git a/packages/das/src/queue/constants.ts b/packages/das/src/queue/constants.ts index 9151658..fd77f4e 100644 --- a/packages/das/src/queue/constants.ts +++ b/packages/das/src/queue/constants.ts @@ -7,3 +7,14 @@ export const FETCH_JOBS = { } as const; export const DEFAULT_BACKFILL_DAYS = 40; + +export function prFilesJobId( + repoFullName: string, + prNumber: number, + headSha: string | null, + baseSha: string | null, +): string { + return `files-${repoFullName}-${prNumber}-${headSha ?? "no-head"}-${ + baseSha ?? "no-base" + }`; +} diff --git a/packages/das/src/queue/fetch.processor.ts b/packages/das/src/queue/fetch.processor.ts index 5449e3b..27860d2 100644 --- a/packages/das/src/queue/fetch.processor.ts +++ b/packages/das/src/queue/fetch.processor.ts @@ -1,11 +1,16 @@ import { Processor, WorkerHost, InjectQueue } from "@nestjs/bullmq"; import { Logger } from "@nestjs/common"; import { InjectRepository } from "@nestjs/typeorm"; -import { Repository } from "typeorm"; +import { IsNull, Repository } from "typeorm"; import { Job, Queue } from "bullmq"; import { Issue, PullRequest } from "../entities"; import { GitHubFetcherService } from "../webhook/github-fetcher.service"; -import { FETCH_QUEUE, FETCH_JOBS, DEFAULT_BACKFILL_DAYS } from "./constants"; +import { + FETCH_QUEUE, + FETCH_JOBS, + DEFAULT_BACKFILL_DAYS, + prFilesJobId, +} from "./constants"; export interface PrMetadataJobData { repoFullName: string; @@ -15,6 +20,8 @@ export interface PrMetadataJobData { export interface PrFilesJobData { repoFullName: string; prNumber: number; + expectedHeadSha?: string | null; + expectedBaseSha?: string | null; } export interface BackfillRepoJobData { @@ -22,6 +29,11 @@ export interface BackfillRepoJobData { days?: number; } +interface PrFilesGeneration { + headSha: string | null; + baseSha: string | null; +} + type JobData = PrMetadataJobData | PrFilesJobData | BackfillRepoJobData; @Processor(FETCH_QUEUE, { concurrency: 5 }) @@ -48,8 +60,7 @@ export class FetchProcessor extends WorkerHost { break; } case FETCH_JOBS.PR_FILES: { - const { repoFullName, prNumber } = job.data as PrFilesJobData; - await this.handlePrFiles(repoFullName, prNumber); + await this.handlePrFiles(job.data as PrFilesJobData); break; } case FETCH_JOBS.BACKFILL_REPO: { @@ -92,18 +103,25 @@ export class FetchProcessor extends WorkerHost { } } - private async handlePrFiles( - repoFullName: string, - prNumber: number, - ): Promise { + private async handlePrFiles(data: PrFilesJobData): Promise { + const { repoFullName, prNumber } = data; this.logger.log(`Fetching PR files for ${repoFullName}#${prNumber}`); + const generation = { + headSha: data.expectedHeadSha ?? null, + baseSha: data.expectedBaseSha ?? null, + }; + await this.fetcher.fetchAndStorePrFiles(repoFullName, prNumber); - await this.prRepo.update( - { repoFullName, prNumber }, + const updateResult = await this.prRepo.update( + this.prGenerationCriteria(repoFullName, prNumber, generation), { scoringDataStored: true }, ); + + if (!updateResult.affected) { + await this.handleStalePrFilesJob(repoFullName, prNumber); + } } private async handleBackfill( @@ -122,7 +140,7 @@ export class FetchProcessor extends WorkerHost { this.logger.log(`Backfilled ${prs.length} PRs from ${repoFullName}`); // Enqueue follow-up jobs (metadata + files for every PR). - for (const { prNumber } of prs) { + for (const { prNumber, headSha, baseSha } of prs) { await this.fetchQueue.add( FETCH_JOBS.PR_METADATA, { repoFullName, prNumber }, @@ -135,16 +153,11 @@ export class FetchProcessor extends WorkerHost { }, ); - await this.fetchQueue.add( - FETCH_JOBS.PR_FILES, - { repoFullName, prNumber }, - { - jobId: `files-${repoFullName}-${prNumber}`, - removeOnComplete: true, - removeOnFail: 50, - attempts: 3, - backoff: { type: "exponential", delay: 5000 }, - }, + await this.enqueuePrFilesJob( + repoFullName, + prNumber, + headSha ?? null, + baseSha ?? null, ); } @@ -152,4 +165,61 @@ export class FetchProcessor extends WorkerHost { await this.fetcher.backfillIssues(repoFullName, sinceDate); this.logger.log(`Backfilled issues from ${repoFullName}`); } + + private async handleStalePrFilesJob( + repoFullName: string, + prNumber: number, + ): Promise { + await this.prRepo.update( + { repoFullName, prNumber }, + { scoringDataStored: false }, + ); + + const pr = await this.prRepo.findOneBy({ repoFullName, prNumber }); + if (!pr) return; + + await this.enqueuePrFilesJob( + repoFullName, + prNumber, + pr.headSha ?? null, + pr.baseSha ?? null, + ); + } + + private async enqueuePrFilesJob( + repoFullName: string, + prNumber: number, + expectedHeadSha: string | null, + expectedBaseSha: string | null, + ): Promise { + await this.fetchQueue.add( + FETCH_JOBS.PR_FILES, + { repoFullName, prNumber, expectedHeadSha, expectedBaseSha }, + { + jobId: prFilesJobId( + repoFullName, + prNumber, + expectedHeadSha, + expectedBaseSha, + ), + removeOnComplete: true, + removeOnFail: 50, + attempts: 3, + backoff: { type: "exponential", delay: 5000 }, + }, + ); + } + + private prGenerationCriteria( + repoFullName: string, + prNumber: number, + generation: PrFilesGeneration, + ): Record { + return { + repoFullName, + prNumber, + headSha: generation.headSha ?? IsNull(), + baseSha: generation.baseSha ?? IsNull(), + }; + } } diff --git a/packages/das/src/webhook/github-fetcher.service.ts b/packages/das/src/webhook/github-fetcher.service.ts index d58e18f..ad772c5 100644 --- a/packages/das/src/webhook/github-fetcher.service.ts +++ b/packages/das/src/webhook/github-fetcher.service.ts @@ -636,7 +636,9 @@ export class GitHubFetcherService implements OnModuleInit { async backfillPullRequests( repoFullName: string, sinceDate: Date, - ): Promise<{ prNumber: number }[]> { + ): Promise< + { prNumber: number; headSha: string | null; baseSha: string | null }[] + > { const [owner, repo] = repoFullName.split("/"); const token = await this.getTokenForRepo(repoFullName); @@ -718,7 +720,11 @@ export class GitHubFetcherService implements OnModuleInit { } `; - const prs: { prNumber: number }[] = []; + const prs: { + prNumber: number; + headSha: string | null; + baseSha: string | null; + }[] = []; let cursor: string | null = null; let defaultBranchWritten = false; @@ -823,7 +829,11 @@ export class GitHubFetcherService implements OnModuleInit { pr.timelineItems?.nodes ?? [], ); - prs.push({ prNumber: pr.number }); + prs.push({ + prNumber: pr.number, + headSha: pr.headRefOid ?? null, + baseSha: pr.baseRefOid ?? null, + }); } if (shouldStop || !page.pageInfo.hasNextPage) break; diff --git a/packages/das/src/webhook/handlers/pull-request.handler.ts b/packages/das/src/webhook/handlers/pull-request.handler.ts index 3406a83..7213834 100644 --- a/packages/das/src/webhook/handlers/pull-request.handler.ts +++ b/packages/das/src/webhook/handlers/pull-request.handler.ts @@ -5,7 +5,7 @@ import { InjectQueue } from "@nestjs/bullmq"; import { Repository } from "typeorm"; import { Queue } from "bullmq"; import { PullRequest, Repo } from "../../entities"; -import { FETCH_QUEUE, FETCH_JOBS } from "../../queue/constants"; +import { FETCH_QUEUE, FETCH_JOBS, prFilesJobId } from "../../queue/constants"; @Injectable() export class PullRequestHandler { @@ -96,10 +96,17 @@ export class PullRequestHandler { ); } - const jobId = `files-${repoFullName}-${prNumber}`; + const expectedHeadSha = data.headSha ?? null; + const expectedBaseSha = data.baseSha ?? null; + const jobId = prFilesJobId( + repoFullName, + prNumber, + expectedHeadSha, + expectedBaseSha, + ); await this.fetchQueue.add( FETCH_JOBS.PR_FILES, - { repoFullName, prNumber }, + { repoFullName, prNumber, expectedHeadSha, expectedBaseSha }, { jobId, removeOnComplete: true, From 92b3c3b928feba36acb68f23e4b75021ed1c3b3b Mon Sep 17 00:00:00 2001 From: MkDev11 <94194147+MkDev11@users.noreply.github.com> Date: Tue, 12 May 2026 09:56:18 -0700 Subject: [PATCH 06/15] Fix label actor role evidence (#14) * Fix label actor role evidence * Run tests before DAS build * Drop contributor DB view tests --------- Co-authored-by: mkdev11 --- .../das/src/webhook/github-fetcher.service.ts | 4 +- .../das/src/webhook/handlers/label.handler.ts | 4 +- .../db/20_view_contributor_repo_roles.sql | 63 +++++++++++++++++-- packages/db/24_view_pr_labels_by_actor.sql | 4 +- 4 files changed, 65 insertions(+), 10 deletions(-) diff --git a/packages/das/src/webhook/github-fetcher.service.ts b/packages/das/src/webhook/github-fetcher.service.ts index ad772c5..b902f83 100644 --- a/packages/das/src/webhook/github-fetcher.service.ts +++ b/packages/das/src/webhook/github-fetcher.service.ts @@ -982,8 +982,8 @@ export class GitHubFetcherService implements OnModuleInit { /** * Upsert a list of LABELED_EVENT / UNLABELED_EVENT timeline nodes into * the label_events table. Actor role is resolved at read time via - * contributor_repo_roles — GraphQL's actor type doesn't expose - * authorAssociation. + * contributor_repo_roles using stored PR/issue, review, and comment + * association evidence; GraphQL's actor type doesn't expose authorAssociation. */ private async saveLabelTimelineEvents( repoFullName: string, diff --git a/packages/das/src/webhook/handlers/label.handler.ts b/packages/das/src/webhook/handlers/label.handler.ts index 4a4eaaf..5304fa1 100644 --- a/packages/das/src/webhook/handlers/label.handler.ts +++ b/packages/das/src/webhook/handlers/label.handler.ts @@ -34,8 +34,8 @@ export class LabelHandler { source === "pr" ? payload.pull_request.number : payload.issue.number; // Append to label_events log. Actor's repo role is resolved at read time - // via contributor_repo_roles (see pr_labels_by_actor view) — neither the - // webhook sender nor GraphQL LabeledEvent.actor expose author_association. + // via contributor_repo_roles using stored PR/issue, review, and comment + // association evidence; label actors themselves don't expose it. await this.labelEventRepo.save({ repoFullName, targetNumber, diff --git a/packages/db/20_view_contributor_repo_roles.sql b/packages/db/20_view_contributor_repo_roles.sql index eb0f13f..1e084df 100644 --- a/packages/db/20_view_contributor_repo_roles.sql +++ b/packages/db/20_view_contributor_repo_roles.sql @@ -1,5 +1,8 @@ -- Latest known association per contributor per repo. --- Unions PRs and issues, takes the most recently created record. +-- Uses every table that stores GitHub's author_association/reviewer_association: +-- PR authors, issue authors, submitted reviews, and issue/PR thread comments. +-- Rows without a stored association are ignored; label views should use the +-- latest known role, not let a missing observation erase earlier evidence. CREATE OR REPLACE VIEW contributor_repo_roles AS SELECT DISTINCT ON (repo_full_name, author_github_id) @@ -8,10 +11,62 @@ SELECT DISTINCT ON (repo_full_name, author_github_id) author_login, author_association FROM ( - SELECT repo_full_name, author_github_id, author_login, author_association, created_at + SELECT + repo_full_name, + author_github_id, + author_login, + author_association, + created_at AS observed_at, + 10 AS source_rank, + 'pr:' || pr_number::text AS source_key FROM pull_requests + WHERE author_github_id IS NOT NULL + AND author_github_id <> '' + AND author_association IS NOT NULL + UNION ALL - SELECT repo_full_name, author_github_id, author_login, author_association, created_at + + SELECT + repo_full_name, + author_github_id, + author_login, + author_association, + created_at AS observed_at, + 10 AS source_rank, + 'issue:' || issue_number::text AS source_key FROM issues + WHERE author_github_id IS NOT NULL + AND author_github_id <> '' + AND author_association IS NOT NULL + + UNION ALL + + SELECT + repo_full_name, + reviewer_github_id AS author_github_id, + reviewer_login AS author_login, + reviewer_association AS author_association, + submitted_at AS observed_at, + 20 AS source_rank, + 'review:' || pr_number::text || ':' || submitted_at::text AS source_key + FROM reviews + WHERE reviewer_github_id IS NOT NULL + AND reviewer_github_id <> '' + AND reviewer_association IS NOT NULL + + UNION ALL + + SELECT + repo_full_name, + author_github_id, + author_login, + author_association, + COALESCE(updated_at, created_at) AS observed_at, + 30 AS source_rank, + 'comment:' || comment_id::text AS source_key + FROM comments + WHERE author_github_id IS NOT NULL + AND author_github_id <> '' + AND author_association IS NOT NULL ) combined -ORDER BY repo_full_name, author_github_id, created_at DESC; +ORDER BY repo_full_name, author_github_id, observed_at DESC, source_rank DESC, source_key DESC; diff --git a/packages/db/24_view_pr_labels_by_actor.sql b/packages/db/24_view_pr_labels_by_actor.sql index d6cbc88..8b10f27 100644 --- a/packages/db/24_view_pr_labels_by_actor.sql +++ b/packages/db/24_view_pr_labels_by_actor.sql @@ -2,8 +2,8 @@ -- Collapses label_events to the latest action per (repo, pr, label); only rows -- where the latest action was "labeled" are included (i.e. label still applied). -- actor_association is resolved from contributor_repo_roles (the actor's most --- recently observed role from PRs/issues they've authored in this repo). --- Actors who've never authored anything return NULL for actor_association. +-- recently observed role from authored PRs/issues, reviews, or comments in +-- this repo). Actors with no stored association evidence return NULL. CREATE OR REPLACE VIEW pr_labels_by_actor AS WITH latest_events AS ( From cd3e26ef85988f2507a0008d1f6ef3e35bfa506e Mon Sep 17 00:00:00 2001 From: MkDev11 <94194147+MkDev11@users.noreply.github.com> Date: Tue, 12 May 2026 11:18:02 -0700 Subject: [PATCH 07/15] Fetch base content for deleted PR files (#21) * Fetch base content for deleted PR files * Tighten deleted file content regression * Strengthen deleted file content coverage * Drop contributor fetcher test --------- Co-authored-by: mkdev11 --- .../das/src/webhook/github-fetcher.service.ts | 27 ++++++++++++------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/packages/das/src/webhook/github-fetcher.service.ts b/packages/das/src/webhook/github-fetcher.service.ts index b902f83..88ad900 100644 --- a/packages/das/src/webhook/github-fetcher.service.ts +++ b/packages/das/src/webhook/github-fetcher.service.ts @@ -477,15 +477,19 @@ export class GitHubFetcherService implements OnModuleInit { headSha: string, baseSha: string | null, ): Promise { - // Only fetch contents for files that have a meaningful version to fetch - const scored = files.filter((f) => f.status !== "removed"); - if (scored.length === 0) return; + // Added files have only a head blob; removed files have only a base blob. + // Keep removed files when a base SHA is available so deletion scoring has + // the content that existed before the PR. + const contentFiles = files.filter( + (f) => f.status !== "removed" || baseSha !== null, + ); + if (contentFiles.length === 0) return; let batchSize = GRAPHQL_FILES_BATCH_SIZE; const minBatchSize = 5; - for (let i = 0; i < scored.length; ) { - const batch = scored.slice(i, i + batchSize); + for (let i = 0; i < contentFiles.length; ) { + const batch = contentFiles.slice(i, i + batchSize); try { await this.fetchContentBatch( repoFullName, @@ -537,11 +541,14 @@ export class GitHubFetcherService implements OnModuleInit { `base${i}: object(expression: "${baseExpr}") { ... on Blob { text byteSize isBinary } }`, ); } - // Head version (already filtered out removed files at caller) - const headExpr = this.escapeGraphql(`${headSha}:${file.filename}`); - fields.push( - `head${i}: object(expression: "${headExpr}") { ... on Blob { text byteSize isBinary } }`, - ); + // Removed files do not exist at head; store a null headContent while + // still fetching the base blob above. + if (file.status !== "removed") { + const headExpr = this.escapeGraphql(`${headSha}:${file.filename}`); + fields.push( + `head${i}: object(expression: "${headExpr}") { ... on Blob { text byteSize isBinary } }`, + ); + } } const query = ` From 44604b5b431067edd4ce82ad05b01066b545d7fa Mon Sep 17 00:00:00 2001 From: Landyn Date: Tue, 12 May 2026 16:22:48 -0500 Subject: [PATCH 08/15] ci: add pr-source-check workflow (#57) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Branch protection on main requires a 'pr-source-check' status, but no workflow was publishing it — leaving PRs into main perma-blocked. Mirror the workflow used in entrius/gittensor and entrius/gittensor-ui: enforce that PRs into main originate from 'test'. --- .github/workflows/pr-source-check.yml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 .github/workflows/pr-source-check.yml diff --git a/.github/workflows/pr-source-check.yml b/.github/workflows/pr-source-check.yml new file mode 100644 index 0000000..42a6c29 --- /dev/null +++ b/.github/workflows/pr-source-check.yml @@ -0,0 +1,18 @@ +name: pr-source-check + +on: + pull_request: + branches: [main] + +jobs: + pr-source-check: + runs-on: ubuntu-latest + steps: + - name: Enforce source = test for PRs into main + run: | + if [ "${{ github.event.pull_request.head.ref }}" != "test" ]; then + echo "::error::PRs into main must originate from 'test'. Head is '${{ github.event.pull_request.head.ref }}'." + echo "::error::Open your PR against 'test' instead. main only advances via the Release workflow." + exit 1 + fi + echo "PR source is 'test' — allowed." From f5b11c16b417b6ab3972e3d65c65efe23c588cfd Mon Sep 17 00:00:00 2001 From: Hunnyboy1217 <110440428+hunnyboy1217@users.noreply.github.com> Date: Wed, 13 May 2026 14:57:16 -0700 Subject: [PATCH 09/15] Make label_events writes idempotent on backfill (#25) (#26) * Make label_events writes idempotent on backfill (#25) * Move unique index into 07_label_events.sql, drop migration files --- .../das/src/webhook/github-fetcher.service.ts | 29 +++++++++------- .../das/src/webhook/handlers/label.handler.ts | 33 ++++++++++++------- packages/db/07_label_events.sql | 5 +++ 3 files changed, 44 insertions(+), 23 deletions(-) diff --git a/packages/das/src/webhook/github-fetcher.service.ts b/packages/das/src/webhook/github-fetcher.service.ts index 88ad900..889c47b 100644 --- a/packages/das/src/webhook/github-fetcher.service.ts +++ b/packages/das/src/webhook/github-fetcher.service.ts @@ -987,8 +987,10 @@ export class GitHubFetcherService implements OnModuleInit { } /** - * Upsert a list of LABELED_EVENT / UNLABELED_EVENT timeline nodes into - * the label_events table. Actor role is resolved at read time via + * Insert LABELED_EVENT / UNLABELED_EVENT timeline nodes into label_events. + * Idempotent: relies on the uq_label_events_natural_key UNIQUE index so + * re-running backfill (or BullMQ retries) collapses to a no-op for events + * already written. Actor role is resolved at read time via * contributor_repo_roles using stored PR/issue, review, and comment * association evidence; GraphQL's actor type doesn't expose authorAssociation. */ @@ -998,23 +1000,28 @@ export class GitHubFetcherService implements OnModuleInit { targetType: "pr" | "issue", nodes: any[], ): Promise { - for (const node of nodes) { - if (!node || !node.label?.name || !node.createdAt) continue; - const action = - node.__typename === "LabeledEvent" ? "labeled" : "unlabeled"; - - await this.labelEventRepo.save({ + const rows = nodes + .filter((node) => node && node.label?.name && node.createdAt) + .map((node) => ({ repoFullName, targetNumber, targetType, labelName: node.label.name, - action, + action: node.__typename === "LabeledEvent" ? "labeled" : "unlabeled", actorGithubId: node.actor?.databaseId ? String(node.actor.databaseId) : null, actorLogin: node.actor?.login ?? null, timestamp: node.createdAt, - }); - } + })); + + if (rows.length === 0) return; + + await this.labelEventRepo + .createQueryBuilder() + .insert() + .values(rows) + .orIgnore() + .execute(); } } diff --git a/packages/das/src/webhook/handlers/label.handler.ts b/packages/das/src/webhook/handlers/label.handler.ts index 5304fa1..ab813bb 100644 --- a/packages/das/src/webhook/handlers/label.handler.ts +++ b/packages/das/src/webhook/handlers/label.handler.ts @@ -34,18 +34,27 @@ export class LabelHandler { source === "pr" ? payload.pull_request.number : payload.issue.number; // Append to label_events log. Actor's repo role is resolved at read time - // via contributor_repo_roles using stored PR/issue, review, and comment - // association evidence; label actors themselves don't expose it. - await this.labelEventRepo.save({ - repoFullName, - targetNumber, - targetType: source, - labelName: label.name, - action, - actorGithubId: sender ? String(sender.id) : null, - actorLogin: sender?.login ?? null, - timestamp: new Date().toISOString(), - }); + // via contributor_repo_roles (see pr_labels_by_actor view) using stored + // PR/issue, review, and comment association evidence — neither the webhook + // sender nor GraphQL LabeledEvent.actor expose author_association. + // orIgnore() makes the insert idempotent under the uq_label_events_natural_key + // constraint; same-delivery retries are already gated upstream by + // webhook_deliveries, this is defense-in-depth. + await this.labelEventRepo + .createQueryBuilder() + .insert() + .values({ + repoFullName, + targetNumber, + targetType: source, + labelName: label.name, + action, + actorGithubId: sender ? String(sender.id) : null, + actorLogin: sender?.login ?? null, + timestamp: new Date().toISOString(), + }) + .orIgnore() + .execute(); // Update current labels snapshot on the parent row const currentLabels: string[] = diff --git a/packages/db/07_label_events.sql b/packages/db/07_label_events.sql index 69c3730..8c2374f 100644 --- a/packages/db/07_label_events.sql +++ b/packages/db/07_label_events.sql @@ -16,3 +16,8 @@ CREATE TABLE IF NOT EXISTS label_events ( ); CREATE INDEX IF NOT EXISTS idx_label_events_target ON label_events(repo_full_name, target_number, timestamp); + +CREATE UNIQUE INDEX IF NOT EXISTS uq_label_events_natural_key + ON label_events (repo_full_name, target_number, target_type, + label_name, action, timestamp) + NULLS NOT DISTINCT; From 25c37d92158d7f14d84739edb9729a1890625df5 Mon Sep 17 00:00:00 2001 From: Desel72 <6442298+Desel72@users.noreply.github.com> Date: Wed, 13 May 2026 17:24:19 -0500 Subject: [PATCH 10/15] fix: persist transferred issue state (#34) --- packages/das/src/webhook/github-fetcher.service.ts | 14 +++++++++++++- packages/das/src/webhook/handlers/issue.handler.ts | 4 ++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/packages/das/src/webhook/github-fetcher.service.ts b/packages/das/src/webhook/github-fetcher.service.ts index 889c47b..c5a80c1 100644 --- a/packages/das/src/webhook/github-fetcher.service.ts +++ b/packages/das/src/webhook/github-fetcher.service.ts @@ -884,11 +884,14 @@ export class GitHubFetcherService implements OnModuleInit { authorAssociation labels(first: 10) { nodes { name } } timelineItems( - itemTypes: [LABELED_EVENT, UNLABELED_EVENT] + itemTypes: [LABELED_EVENT, UNLABELED_EVENT, TRANSFERRED_EVENT] first: 30 ) { nodes { __typename + ... on TransferredEvent { + createdAt + } ... on LabeledEvent { createdAt label { name } @@ -966,6 +969,15 @@ export class GitHubFetcherService implements OnModuleInit { ), }; + if ( + (issue.timelineItems?.nodes ?? []).some( + (node: { __typename?: string }) => + node.__typename === "TransferredEvent", + ) + ) { + issueData.isTransferred = true; + } + if (issue.state === "OPEN") { issueData.solvedByPr = null; } diff --git a/packages/das/src/webhook/handlers/issue.handler.ts b/packages/das/src/webhook/handlers/issue.handler.ts index a94edbc..ac54368 100644 --- a/packages/das/src/webhook/handlers/issue.handler.ts +++ b/packages/das/src/webhook/handlers/issue.handler.ts @@ -40,6 +40,10 @@ export class IssueHandler { data.solvedByPr = null; } + if (payload.action === "transferred") { + data.isTransferred = true; + } + // The `edited` action fires specifically for body or title changes. // Use the webhook's updated_at as the precise edit timestamp — for // other actions (labeled, closed, commented, etc.) don't touch From 4d8121df46dd00d70407eef9dfe5eaf4110b9b69 Mon Sep 17 00:00:00 2001 From: Desel72 <6442298+Desel72@users.noreply.github.com> Date: Wed, 13 May 2026 17:26:25 -0500 Subject: [PATCH 11/15] fix(mirror): filter cross-repo closing refs (#36) --- .../das/src/webhook/github-fetcher.service.ts | 29 +++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/packages/das/src/webhook/github-fetcher.service.ts b/packages/das/src/webhook/github-fetcher.service.ts index c5a80c1..7f3adfd 100644 --- a/packages/das/src/webhook/github-fetcher.service.ts +++ b/packages/das/src/webhook/github-fetcher.service.ts @@ -20,6 +20,11 @@ interface InstallationToken { expiresAt: number; } +interface ClosingIssueReference { + number?: number; + repository?: { nameWithOwner?: string } | null; +} + // Files larger than this are stored with null content (AST parsing is wasteful past this). const MAX_FILE_SIZE_BYTES = 1_000_000; @@ -280,7 +285,10 @@ export class GitHubFetcherService implements OnModuleInit { bodyText lastEditedAt closingIssuesReferences(first: 10) { - nodes { number } + nodes { + number + repository { nameWithOwner } + } } } } @@ -310,12 +318,29 @@ export class GitHubFetcherService implements OnModuleInit { const nodes = pr.closingIssuesReferences?.nodes ?? []; return { - closingIssueNumbers: nodes.map((n: { number: number }) => n.number), + closingIssueNumbers: this.sameRepoClosingIssueNumbers( + repoFullName, + nodes, + ), body: pr.bodyText ?? null, lastEditedAt: pr.lastEditedAt ?? null, }; } + private sameRepoClosingIssueNumbers( + repoFullName: string, + nodes: ClosingIssueReference[], + ): number[] { + const expectedRepo = repoFullName.toLowerCase(); + return nodes + .filter( + (node) => + node.repository?.nameWithOwner?.toLowerCase() === expectedRepo, + ) + .map((node) => node.number) + .filter((number): number is number => typeof number === "number"); + } + // --- PR files + contents (REST for list, batched GraphQL for contents) --- /** From 8cf0853d5de1823064120e06b903feb0d78b86b8 Mon Sep 17 00:00:00 2001 From: Desel72 <6442298+Desel72@users.noreply.github.com> Date: Wed, 13 May 2026 21:08:57 -0500 Subject: [PATCH 12/15] fix: fail incomplete pr content fetches (#38) --- packages/das/src/webhook/github-fetcher.service.ts | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/packages/das/src/webhook/github-fetcher.service.ts b/packages/das/src/webhook/github-fetcher.service.ts index 7f3adfd..967e057 100644 --- a/packages/das/src/webhook/github-fetcher.service.ts +++ b/packages/das/src/webhook/github-fetcher.service.ts @@ -405,10 +405,9 @@ export class GitHubFetcherService implements OnModuleInit { // 3. Fetch file contents in batches (base + head in one GraphQL call each) if (!pr.headSha) { - this.logger.warn( - `PR ${repoFullName}#${prNumber} has no head SHA — skipping content fetch`, + throw new Error( + `PR ${repoFullName}#${prNumber} has no head SHA; cannot fetch content`, ); - return; } // Prefer merge-base SHA (true common ancestor) over base SHA for @@ -536,10 +535,9 @@ export class GitHubFetcherService implements OnModuleInit { batchSize = newSize; // Retry same i with smaller batch } else { - this.logger.warn( - `GraphQL content batch failed at min size ${minBatchSize}: ${err}. Skipping batch.`, + throw new Error( + `GraphQL content batch failed at min size ${minBatchSize}: ${err}`, ); - i += batch.length; } } } From dd7ee4cacfa82727b15cf32c5a62c7893e1ff343 Mon Sep 17 00:00:00 2001 From: Ander <61125407+anderdc@users.noreply.github.com> Date: Wed, 13 May 2026 21:26:15 -0500 Subject: [PATCH 13/15] feat(dashboard): add slim issues endpoint to replace per-miner fan-out (#93) * feat(dashboard): add slim issues endpoint to replace per-miner fan-out New DashboardModule exposing GET /api/v1/dashboard/issues?since= for the gittensor-ui dashboard's trend chart and Issues Solved KPI. Returns slim issue rows (one bulk call) so the UI no longer fans out N parallel /miners//issues requests on every dashboard mount. The mirror is roster-blind by design: every issue is returned regardless of author. The UI blends with the gittensor miner roster client-side to filter to subnet authors. Validator-facing /miners//issues is unchanged. * style(api.module): prettier formatting --------- Co-authored-by: anderdc --- packages/das/src/api/api.module.ts | 10 +++- .../src/api/dashboard/dashboard.controller.ts | 37 ++++++++++++ .../src/api/dashboard/dashboard.service.ts | 58 +++++++++++++++++++ 3 files changed, 104 insertions(+), 1 deletion(-) create mode 100644 packages/das/src/api/dashboard/dashboard.controller.ts create mode 100644 packages/das/src/api/dashboard/dashboard.service.ts diff --git a/packages/das/src/api/api.module.ts b/packages/das/src/api/api.module.ts index 2f8dea2..14b6fcc 100644 --- a/packages/das/src/api/api.module.ts +++ b/packages/das/src/api/api.module.ts @@ -13,6 +13,8 @@ import { FETCH_QUEUE } from "../queue/constants"; import { AdminController } from "./admin.controller"; import { RequireApiKeyGuard } from "./require-api-key.guard"; import { HealthController } from "./health.controller"; +import { DashboardController } from "./dashboard/dashboard.controller"; +import { DashboardService } from "./dashboard/dashboard.service"; import { MinersController } from "./miners/miners.controller"; import { MinersService } from "./miners/miners.service"; import { PullsController } from "./pulls/pulls.controller"; @@ -31,11 +33,17 @@ import { PullsService } from "./pulls/pulls.service"; BullModule.registerQueue({ name: FETCH_QUEUE }), ], controllers: [ + DashboardController, MinersController, PullsController, AdminController, HealthController, ], - providers: [MinersService, PullsService, RequireApiKeyGuard], + providers: [ + DashboardService, + MinersService, + PullsService, + RequireApiKeyGuard, + ], }) export class ApiModule {} diff --git a/packages/das/src/api/dashboard/dashboard.controller.ts b/packages/das/src/api/dashboard/dashboard.controller.ts new file mode 100644 index 0000000..f332206 --- /dev/null +++ b/packages/das/src/api/dashboard/dashboard.controller.ts @@ -0,0 +1,37 @@ +import { BadRequestException, Controller, Get, Query } from "@nestjs/common"; +import { ApiOperation, ApiQuery, ApiTags } from "@nestjs/swagger"; +import { DashboardService } from "./dashboard.service"; + +@ApiTags("Dashboard") +@Controller("api/v1/dashboard") +export class DashboardController { + constructor(private readonly dashboard: DashboardService) {} + + @Get("issues") + @ApiOperation({ + summary: "Slim issue rows for dashboard trend aggregation", + description: + "Returns every issue with `created_at` on or after `since`, plus " + + "every CLOSED issue whose `closed_at` is on or after `since`. " + + "The mirror is intentionally roster-blind: every issue is returned " + + "regardless of author. The dashboard blends with the gittensor API " + + "miner roster client-side to filter to subnet authors. " + + "Designed as a single bulk replacement for the dashboard's per-miner " + + "fan-out against `/miners//issues` (one call instead of N).", + }) + @ApiQuery({ + name: "since", + required: true, + description: "ISO timestamp — earliest creation/close date to include.", + }) + async getIssues(@Query("since") since?: string): Promise { + if (!since) { + throw new BadRequestException("`since` query parameter is required"); + } + const parsed = new Date(since); + if (Number.isNaN(parsed.getTime())) { + throw new BadRequestException("`since` must be a valid ISO timestamp"); + } + return this.dashboard.getIssues(parsed.toISOString()); + } +} diff --git a/packages/das/src/api/dashboard/dashboard.service.ts b/packages/das/src/api/dashboard/dashboard.service.ts new file mode 100644 index 0000000..62a7ab6 --- /dev/null +++ b/packages/das/src/api/dashboard/dashboard.service.ts @@ -0,0 +1,58 @@ +/* eslint-disable @typescript-eslint/no-unsafe-return, @typescript-eslint/no-unsafe-assignment */ +import { Injectable } from "@nestjs/common"; +import { DataSource } from "typeorm"; + +interface DashboardIssueRow { + repo_full_name: string; + issue_number: number; + author_github_id: string | null; + created_at: string; + closed_at: string | null; + state: string; + state_reason: string | null; + solving_pr: { merged_at: string } | null; +} + +@Injectable() +export class DashboardService { + constructor(private readonly dataSource: DataSource) {} + + async getIssues(since: string): Promise<{ + since: string; + generated_at: string; + issues: DashboardIssueRow[]; + }> { + const rows = await this.dataSource.query( + ` + SELECT + LOWER(i.repo_full_name) AS repo_full_name, + i.issue_number, + i.author_github_id, + i.created_at, + i.closed_at, + i.state, + i.state_reason, + ( + SELECT json_build_object('merged_at', sp.merged_at) + FROM pull_requests sp + WHERE sp.repo_full_name = i.repo_full_name + AND sp.pr_number = i.solved_by_pr + AND sp.merged_at IS NOT NULL + LIMIT 1 + ) AS solving_pr + FROM issues i + WHERE + i.created_at >= $1 + OR (i.state = 'CLOSED' AND i.closed_at >= $1) + ORDER BY i.created_at DESC + `, + [since], + ); + + return { + since, + generated_at: new Date().toISOString(), + issues: rows as DashboardIssueRow[], + }; + } +} From 4a3a825c738cbcdce5df028b3e6b207ec7234f65 Mon Sep 17 00:00:00 2001 From: volcano303 <75143900+volcano303@users.noreply.github.com> Date: Wed, 13 May 2026 16:36:12 -1000 Subject: [PATCH 14/15] fix(webhook): refresh repo last event on review and comment events (#51) --- packages/das/src/webhook/handlers/comment.handler.ts | 11 ++++++++++- .../src/webhook/handlers/review-comment.handler.ts | 11 ++++++++++- packages/das/src/webhook/handlers/review.handler.ts | 8 +++++++- 3 files changed, 27 insertions(+), 3 deletions(-) diff --git a/packages/das/src/webhook/handlers/comment.handler.ts b/packages/das/src/webhook/handlers/comment.handler.ts index ee104ea..d5760e7 100644 --- a/packages/das/src/webhook/handlers/comment.handler.ts +++ b/packages/das/src/webhook/handlers/comment.handler.ts @@ -2,13 +2,15 @@ import { Injectable } from "@nestjs/common"; import { InjectRepository } from "@nestjs/typeorm"; import { Repository } from "typeorm"; -import { Comment } from "../../entities"; +import { Comment, Repo } from "../../entities"; @Injectable() export class CommentHandler { constructor( @InjectRepository(Comment) private readonly commentRepo: Repository, + @InjectRepository(Repo) + private readonly repoRepo: Repository, ) {} async handle(payload: Record): Promise { @@ -20,6 +22,9 @@ export class CommentHandler { repoFullName, commentId: String(comment.id), }); + await this.repoRepo.update(repoFullName, { + lastEventAt: new Date().toISOString(), + }); return; } @@ -40,5 +45,9 @@ export class CommentHandler { }; await this.commentRepo.upsert(data, ["repoFullName", "commentId"]); + + await this.repoRepo.update(repoFullName, { + lastEventAt: new Date().toISOString(), + }); } } diff --git a/packages/das/src/webhook/handlers/review-comment.handler.ts b/packages/das/src/webhook/handlers/review-comment.handler.ts index 3b957da..0b9b350 100644 --- a/packages/das/src/webhook/handlers/review-comment.handler.ts +++ b/packages/das/src/webhook/handlers/review-comment.handler.ts @@ -2,13 +2,15 @@ import { Injectable } from "@nestjs/common"; import { InjectRepository } from "@nestjs/typeorm"; import { Repository } from "typeorm"; -import { ReviewComment } from "../../entities"; +import { Repo, ReviewComment } from "../../entities"; @Injectable() export class ReviewCommentHandler { constructor( @InjectRepository(ReviewComment) private readonly reviewCommentRepo: Repository, + @InjectRepository(Repo) + private readonly repoRepo: Repository, ) {} async handle(payload: Record): Promise { @@ -20,6 +22,9 @@ export class ReviewCommentHandler { repoFullName, commentId: String(comment.id), }); + await this.repoRepo.update(repoFullName, { + lastEventAt: new Date().toISOString(), + }); return; } @@ -41,5 +46,9 @@ export class ReviewCommentHandler { }; await this.reviewCommentRepo.upsert(data, ["repoFullName", "commentId"]); + + await this.repoRepo.update(repoFullName, { + lastEventAt: new Date().toISOString(), + }); } } diff --git a/packages/das/src/webhook/handlers/review.handler.ts b/packages/das/src/webhook/handlers/review.handler.ts index a2f14af..3a5f573 100644 --- a/packages/das/src/webhook/handlers/review.handler.ts +++ b/packages/das/src/webhook/handlers/review.handler.ts @@ -2,13 +2,15 @@ import { Injectable } from "@nestjs/common"; import { InjectRepository } from "@nestjs/typeorm"; import { Repository } from "typeorm"; -import { Review } from "../../entities"; +import { Repo, Review } from "../../entities"; @Injectable() export class ReviewHandler { constructor( @InjectRepository(Review) private readonly reviewRepo: Repository, + @InjectRepository(Repo) + private readonly repoRepo: Repository, ) {} async handle(payload: Record): Promise { @@ -34,5 +36,9 @@ export class ReviewHandler { "reviewerGithubId", "submittedAt", ]); + + await this.repoRepo.update(repoFullName, { + lastEventAt: new Date().toISOString(), + }); } } From c84bad0262a54201cd3870159a9b6707c20f5051 Mon Sep 17 00:00:00 2001 From: Ander <61125407+anderdc@users.noreply.github.com> Date: Sat, 16 May 2026 14:25:28 -0500 Subject: [PATCH 15/15] feat(api): add repo maintainers endpoint (#104) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add GET /api/v1/repos/:owner/:repo/maintainers, returning users whose latest known association for the repo is OWNER, MEMBER, or COLLABORATOR. Reads the existing contributor_repo_roles view — no new table or GitHub API permission. An unknown repo returns an empty maintainers list. Consumed by the gittensor validator to route the per-repo maintainer_cut emission carve-out. Co-authored-by: anderdc --- packages/das/src/api/api.module.ts | 4 ++ .../das/src/api/repos/repos.controller.ts | 27 ++++++++++++ packages/das/src/api/repos/repos.service.ts | 41 +++++++++++++++++++ 3 files changed, 72 insertions(+) create mode 100644 packages/das/src/api/repos/repos.controller.ts create mode 100644 packages/das/src/api/repos/repos.service.ts diff --git a/packages/das/src/api/api.module.ts b/packages/das/src/api/api.module.ts index 14b6fcc..c6cb2a0 100644 --- a/packages/das/src/api/api.module.ts +++ b/packages/das/src/api/api.module.ts @@ -19,6 +19,8 @@ import { MinersController } from "./miners/miners.controller"; import { MinersService } from "./miners/miners.service"; import { PullsController } from "./pulls/pulls.controller"; import { PullsService } from "./pulls/pulls.service"; +import { ReposController } from "./repos/repos.controller"; +import { ReposService } from "./repos/repos.service"; @Module({ imports: [ @@ -36,6 +38,7 @@ import { PullsService } from "./pulls/pulls.service"; DashboardController, MinersController, PullsController, + ReposController, AdminController, HealthController, ], @@ -43,6 +46,7 @@ import { PullsService } from "./pulls/pulls.service"; DashboardService, MinersService, PullsService, + ReposService, RequireApiKeyGuard, ], }) diff --git a/packages/das/src/api/repos/repos.controller.ts b/packages/das/src/api/repos/repos.controller.ts new file mode 100644 index 0000000..09314a7 --- /dev/null +++ b/packages/das/src/api/repos/repos.controller.ts @@ -0,0 +1,27 @@ +import { Controller, Get, Param } from "@nestjs/common"; +import { ApiOperation, ApiParam, ApiTags } from "@nestjs/swagger"; +import { ReposService } from "./repos.service"; + +@ApiTags("Repos") +@Controller("api/v1/repos") +export class ReposController { + constructor(private readonly repos: ReposService) {} + + @Get(":owner/:repo/maintainers") + @ApiOperation({ + summary: "Maintainer-role contributors for a repo", + description: + "Returns users whose latest known GitHub association for the repo " + + "is OWNER, MEMBER, or COLLABORATOR, synthesized from PR/issue/" + + "review/comment activity (contributor_repo_roles view). An unknown " + + "repo returns an empty maintainers list, not a 404.", + }) + @ApiParam({ name: "owner", description: "Repository owner (org or user)" }) + @ApiParam({ name: "repo", description: "Repository name" }) + async getMaintainers( + @Param("owner") owner: string, + @Param("repo") repo: string, + ): Promise { + return this.repos.getMaintainers(owner, repo); + } +} diff --git a/packages/das/src/api/repos/repos.service.ts b/packages/das/src/api/repos/repos.service.ts new file mode 100644 index 0000000..18e3b9c --- /dev/null +++ b/packages/das/src/api/repos/repos.service.ts @@ -0,0 +1,41 @@ +/* eslint-disable @typescript-eslint/no-unsafe-return, @typescript-eslint/no-unsafe-assignment */ +import { Injectable } from "@nestjs/common"; +import { DataSource } from "typeorm"; + +@Injectable() +export class ReposService { + constructor(private readonly dataSource: DataSource) {} + + async getMaintainers( + owner: string, + repo: string, + ): Promise<{ + repo_full_name: string; + generated_at: string; + maintainers: unknown[]; + }> { + const repoFullName = `${owner}/${repo}`; + + // The association literals must stay in sync with gittensor + // constants.py MAINTAINER_ASSOCIATIONS. + const rows = await this.dataSource.query( + ` + SELECT + cr.author_github_id AS github_id, + cr.author_login AS login, + cr.author_association AS association + FROM contributor_repo_roles cr + WHERE LOWER(cr.repo_full_name) = LOWER($1) + AND cr.author_association IN ('OWNER', 'MEMBER', 'COLLABORATOR') + ORDER BY cr.author_github_id + `, + [repoFullName], + ); + + return { + repo_full_name: repoFullName.toLowerCase(), + generated_at: new Date().toISOString(), + maintainers: rows, + }; + } +}