From 850f41c260a73b5cf45a19d256a7782d2e7b09a9 Mon Sep 17 00:00:00 2001 From: Karan Lokchandani Date: Fri, 31 Jul 2026 19:12:43 +0530 Subject: [PATCH 1/2] feat(cli): add the developer category to search `firecrawl search --categories developer` searches an index built for coding agents: GitHub issues, merged pull requests, repository READMEs, and curated documentation sites. The API already served the category. The CLI rejected the value, and `executeSearch` copied only `web`, `images`, and `news` out of the payload. A developer group therefore never reached the caller. The category is an extra arm, not a filter on the web results, so the API returns its hits in `data.developer`. The command copies that group through, prints it under its own heading, and labels the web group when both are present. A matched passage runs to several KB. The readable output clips it at 500 characters and names `--json`, which keeps the full text. Without the clip, three hits flood a terminal. --- README.md | 8 +- skills/firecrawl-search/SKILL.md | 39 +++++--- src/__tests__/commands/search.test.ts | 122 +++++++++++++++++++++++++- src/commands/search.ts | 46 +++++++++- src/index.ts | 4 +- src/types/search.ts | 19 +++- 6 files changed, 217 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index 4c2c637570..47df8a9ee1 100644 --- a/README.md +++ b/README.md @@ -286,6 +286,9 @@ firecrawl search "web data python" --categories github firecrawl search "transformer architecture" --categories research firecrawl search "machine learning" --categories github,research +# Developer search: GitHub issues, merged PRs, READMEs, and docs +firecrawl search "axum middleware ordering" --categories developer + # Time-based search firecrawl search "AI announcements" --tbs qdr:d # Past day firecrawl search "tech news" --tbs qdr:w # Past week @@ -308,7 +311,7 @@ firecrawl search "AI data tools" | ---------------------------- | ------------------------------------------------------------------------------------------- | | `--limit ` | Maximum results (default: 5, max: 100) | | `--sources ` | Comma-separated: `web`, `images`, `news` (default: web) | -| `--categories ` | Comma-separated: `github`, `research`, `pdf` | +| `--categories ` | Comma-separated: `github`, `research`, `pdf`, `developer` | | `--tbs ` | Time filter: `qdr:h` (hour), `qdr:d` (day), `qdr:w` (week), `qdr:m` (month), `qdr:y` (year) | | `--location ` | Geo-targeting (e.g., "Germany", "San Francisco,California,United States") | | `--country ` | ISO country code (default: US) | @@ -337,6 +340,9 @@ firecrawl search "firecrawl documentation" --scrape --scrape-formats markdown -- # Find research papers firecrawl search "large language models" --categories research --json +# Answer a programming question from issues, merged PRs, READMEs, and docs +firecrawl search "tokio select cancellation safety" --categories developer --json + # Search with location targeting firecrawl search "best coffee shops" --location "Berlin,Germany" --country DE diff --git a/skills/firecrawl-search/SKILL.md b/skills/firecrawl-search/SKILL.md index 87b426cf92..32bed16bd1 100644 --- a/skills/firecrawl-search/SKILL.md +++ b/skills/firecrawl-search/SKILL.md @@ -28,22 +28,37 @@ firecrawl search "your query" --scrape -o .firecrawl/scraped.json --json # News from the past day firecrawl search "your query" --sources news --tbs qdr:d -o .firecrawl/news.json --json + +# Programming question: search GitHub issues, merged PRs, READMEs, and docs +firecrawl search "your query" --categories developer -o .firecrawl/developer.json --json ``` +## Developer search + +`--categories developer` adds an index built for coding agents. It covers GitHub +issues, merged pull requests, repository READMEs, and curated documentation +sites. Use it for a programming question: an error message, an API contract, a +library behaviour, or a known bug. + +The hits arrive in their own `data.developer` group beside `data.web`. Each hit +holds `url`, `title`, and `description`, where `description` is the matched +passage. Read the passages with +`jq -r '.data.developer[] | .url, .description' .firecrawl/developer.json`. + ## Options -| Option | Description | -| ------------------------------------ | --------------------------------------------- | -| `--limit ` | Max number of results | -| `--sources ` | Source types to search | -| `--categories ` | Filter by category | -| `--tbs ` | Time-based search filter | -| `--location` | Location for search results | -| `--country ` | Country code for search | -| `--scrape` | Also scrape full page content for each result | -| `--scrape-formats` | Formats when scraping (default: markdown) | -| `-o, --output ` | Output file path | -| `--json` | Output as JSON | +| Option | Description | +| ---------------------------------------------- | --------------------------------------------- | +| `--limit ` | Max number of results | +| `--sources ` | Source types to search | +| `--categories ` | Filter by category | +| `--tbs ` | Time-based search filter | +| `--location` | Location for search results | +| `--country ` | Country code for search | +| `--scrape` | Also scrape full page content for each result | +| `--scrape-formats` | Formats when scraping (default: markdown) | +| `-o, --output ` | Output file path | +| `--json` | Output as JSON | ## Tips diff --git a/src/__tests__/commands/search.test.ts b/src/__tests__/commands/search.test.ts index 116e128413..aeded653fc 100644 --- a/src/__tests__/commands/search.test.ts +++ b/src/__tests__/commands/search.test.ts @@ -3,11 +3,14 @@ */ import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; -import { executeSearch } from '../../commands/search'; +import { executeSearch, handleSearchCommand } from '../../commands/search'; import { getClient } from '../../utils/client'; import { initializeConfig } from '../../utils/config'; +import { writeOutput } from '../../utils/output'; import { setupTest, teardownTest } from '../utils/mock-client'; +vi.mock('../../utils/output', () => ({ writeOutput: vi.fn() })); + // Mock the Firecrawl client module vi.mock('../../utils/client', async () => { const actual = await vi.importActual('../../utils/client'); @@ -200,6 +203,23 @@ describe('executeSearch', () => { ); }); + it('should include the developer category when provided', async () => { + mockHttpPost.mockResolvedValue(mockSearchResponse({ web: [] })); + + await executeSearch({ + query: 'tokio select cancellation safety', + categories: ['developer'], + }); + + expect(mockHttpPost).toHaveBeenCalledWith( + '/v2/search', + expect.objectContaining({ + query: 'tokio select cancellation safety', + categories: [{ type: 'developer' }], + }) + ); + }); + it('should include multiple categories correctly', async () => { mockHttpPost.mockResolvedValue(mockSearchResponse({ web: [] })); @@ -438,6 +458,28 @@ describe('executeSearch', () => { expect(result.data).toEqual({ web }); }); + it('should return the developer group beside the web results', async () => { + const web = [{ url: 'https://example.com', title: 'Example' }]; + const developer = [ + { + url: 'https://github.com/tokio-rs/tokio/issues/7364', + title: 'select! cancellation safety', + description: 'The matched passage.', + position: 1, + category: 'developer', + }, + ]; + mockHttpPost.mockResolvedValue(mockSearchResponse({ web, developer })); + + const result = await executeSearch({ + query: 'tokio select cancellation safety', + categories: ['developer'], + }); + + expect(result.success).toBe(true); + expect(result.data).toEqual({ web, developer }); + }); + it('should return success result with image results', async () => { const images = [ { @@ -703,10 +745,11 @@ describe('executeSearch', () => { }); it('should accept valid category types', async () => { - const categoryList: Array<'github' | 'research' | 'pdf'> = [ + const categoryList: Array<'github' | 'research' | 'pdf' | 'developer'> = [ 'github', 'research', 'pdf', + 'developer', ]; mockHttpPost.mockResolvedValue(mockSearchResponse({ web: [] })); @@ -740,4 +783,79 @@ describe('executeSearch', () => { } }); }); + + describe('Developer results in the readable output', () => { + // Read the text that `handleSearchCommand` sent to the writer. + const writtenOutput = () => + vi.mocked(writeOutput).mock.calls.at(-1)?.[0] as string; + + it('should label both groups and print each developer hit', async () => { + mockHttpPost.mockResolvedValue( + mockSearchResponse({ + web: [{ url: 'https://example.com', title: 'Example' }], + developer: [ + { + url: 'https://github.com/tokio-rs/tokio/issues/7364', + title: 'select! cancellation safety', + description: 'The matched passage.', + }, + ], + }) + ); + + await handleSearchCommand({ query: 'tokio select cancellation safety' }); + + const output = writtenOutput(); + expect(output).toContain('=== Web Results ==='); + expect(output).toContain('=== Developer Results ==='); + expect(output).toContain('select! cancellation safety'); + expect(output).toContain( + 'URL: https://github.com/tokio-rs/tokio/issues/7364' + ); + expect(output).toContain('The matched passage.'); + }); + + it('should truncate a long passage and name the flag that keeps it', async () => { + const passage = 'x'.repeat(900); + mockHttpPost.mockResolvedValue( + mockSearchResponse({ + developer: [ + { + url: 'https://github.com/firecrawl/firecrawl/issues/1', + title: 'Long passage', + description: passage, + }, + ], + }) + ); + + await handleSearchCommand({ query: 'long passage' }); + + const output = writtenOutput(); + expect(output).toContain('x'.repeat(500)); + expect(output).not.toContain('x'.repeat(501)); + expect(output).toContain('use --json for the full passage'); + }); + + it('should keep the full passage in the JSON output', async () => { + const passage = 'y'.repeat(900); + mockHttpPost.mockResolvedValue( + mockSearchResponse({ + developer: [ + { + url: 'https://github.com/firecrawl/firecrawl/issues/2', + title: 'Long passage', + description: passage, + }, + ], + }) + ); + + await handleSearchCommand({ query: 'long passage', json: true }); + + expect(JSON.parse(writtenOutput()).data.developer[0].description).toBe( + passage + ); + }); + }); }); diff --git a/src/commands/search.ts b/src/commands/search.ts index 9a768ae0e7..1abda30b17 100644 --- a/src/commands/search.ts +++ b/src/commands/search.ts @@ -10,6 +10,7 @@ import type { WebSearchResult, ImageSearchResult, NewsSearchResult, + DeveloperSearchResult, } from '../types/search'; import { getClient, isKeylessMode, keylessRequest } from '../utils/client'; import { writeOutput } from '../utils/output'; @@ -123,6 +124,10 @@ export async function executeSearch( if (payload.web) data.web = payload.web as WebSearchResult[]; if (payload.images) data.images = payload.images as ImageSearchResult[]; if (payload.news) data.news = payload.news as NewsSearchResult[]; + // The `developer` category is an extra arm rather than a filter on the web + // results, so the API returns its hits in their own group. + if (payload.developer) + data.developer = payload.developer as DeveloperSearchResult[]; return { success: true, @@ -139,6 +144,18 @@ export async function executeSearch( } } +/** + * Shorten a matched passage for the human-readable output. A developer passage + * runs to several KB, which floods a terminal. `--json` keeps the full text. + */ +function clipPassage(passage: string): string { + const collapsed = passage.replace(/\s+/g, ' ').trim(); + if (collapsed.length <= 500) { + return collapsed; + } + return `${collapsed.slice(0, 500)}… (truncated, use --json for the full passage)`; +} + /** * Format search data in human-readable way */ @@ -150,7 +167,13 @@ function formatSearchReadable( // Format web results if (data.web && data.web.length > 0) { - if (options.sources && options.sources.length > 1) { + // Label the web group whenever another group follows it, so the reader can + // tell the groups apart. + const hasDeveloperResults = !!data.developer && data.developer.length > 0; + if ( + (options.sources && options.sources.length > 1) || + hasDeveloperResults + ) { lines.push('=== Web Results ==='); lines.push(''); } @@ -179,6 +202,24 @@ function formatSearchReadable( } } + // Format developer results + if (data.developer && data.developer.length > 0) { + if (lines.length > 0) { + lines.push(''); + } + lines.push('=== Developer Results ==='); + lines.push(''); + + for (const result of data.developer) { + lines.push(`${result.title || 'Untitled'}`); + lines.push(` URL: ${result.url}`); + if (result.description) { + lines.push(` ${clipPassage(result.description)}`); + } + lines.push(''); + } + } + // Format image results if (data.images && data.images.length > 0) { if (lines.length > 0) { @@ -253,7 +294,8 @@ export async function handleSearchCommand( const hasResults = (result.data.web && result.data.web.length > 0) || (result.data.images && result.data.images.length > 0) || - (result.data.news && result.data.news.length > 0); + (result.data.news && result.data.news.length > 0) || + (result.data.developer && result.data.developer.length > 0); if (!hasResults) { console.log('No results found.'); diff --git a/src/index.ts b/src/index.ts index b22c8e1a89..9a47d0d57d 100644 --- a/src/index.ts +++ b/src/index.ts @@ -919,7 +919,7 @@ function createSearchCommand(): Command { ) .option( '--categories ', - 'Comma-separated categories to filter: github, research, pdf' + 'Comma-separated categories to filter: github, research, pdf, developer (developer searches indexed GitHub issues, merged PRs, READMEs, and docs)' ) .option( '--tbs ', @@ -1001,7 +1001,7 @@ function createSearchCommand(): Command { .map((c: string) => c.trim().toLowerCase()) as SearchCategory[]; // Validate categories - const validCategories = ['github', 'research', 'pdf']; + const validCategories = ['github', 'research', 'pdf', 'developer']; for (const category of categories) { if (!validCategories.includes(category)) { console.error( diff --git a/src/types/search.ts b/src/types/search.ts index 48ad8295ef..04486bf543 100644 --- a/src/types/search.ts +++ b/src/types/search.ts @@ -5,7 +5,7 @@ import type { ScrapeFormat } from './scrape'; export type SearchSource = 'web' | 'images' | 'news'; -export type SearchCategory = 'github' | 'research' | 'pdf'; +export type SearchCategory = 'github' | 'research' | 'pdf' | 'developer'; export interface SearchOptions { /** Search query (required) */ @@ -18,7 +18,7 @@ export interface SearchOptions { limit?: number; /** Sources to search: web, images, news (default: web) */ sources?: SearchSource[]; - /** Categories to filter results: github, research, pdf */ + /** Categories to filter results: github, research, pdf, developer */ categories?: SearchCategory[]; /** Time-based search parameter (e.g., qdr:h, qdr:d, qdr:w, qdr:m, qdr:y) */ tbs?: string; @@ -98,10 +98,25 @@ export interface NewsSearchResult { }; } +/** + * One hit from the `developer` category. The index covers GitHub issues, + * merged pull requests, repository READMEs, and curated documentation sites. + * `description` holds the matched passage, which runs to several KB. + */ +export interface DeveloperSearchResult { + url: string; + title?: string; + description?: string; + position?: number; + category?: string; +} + export interface SearchResultData { web?: WebSearchResult[]; images?: ImageSearchResult[]; news?: NewsSearchResult[]; + /** Present when the `developer` category is requested. */ + developer?: DeveloperSearchResult[]; } export interface SearchResult { From c5eaa68de35a2e8214fb46afa7a2e3c3851c1620 Mon Sep 17 00:00:00 2001 From: Karan Lokchandani Date: Fri, 31 Jul 2026 21:23:15 +0530 Subject: [PATCH 2/2] feat(cli): add the developer command The developer index now has two surfaces, the same as research: the `developer` category on `firecrawl search`, and a dedicated command. The command mirrors the research commands. It calls GET /v2/developer/search, keeps the keyless free-tier path, and renders the ranked hits as markdown blocks with the passages clipped at 1200 characters; `--json` keeps the full envelope. It takes a query and `--limit` only; the endpoint accepts no filters, and that narrow surface is deliberate. New tests cover the request URL, the readable and JSON output, and the error path. Two argv tests prove the command appears in the root help and parses, so a wrapper cannot drop it the way `executeSearch` once dropped the category group. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01C4vZ6XH65bUJDkLthyfmvC --- README.md | 29 ++++ skills/firecrawl-search/SKILL.md | 12 ++ src/__tests__/cli-argv.test.ts | 26 ++++ src/__tests__/commands/developer.test.ts | 177 +++++++++++++++++++++++ src/commands/developer.ts | 80 ++++++++++ src/index.ts | 48 ++++++ src/types/developer.ts | 17 +++ 7 files changed, 389 insertions(+) create mode 100644 src/__tests__/commands/developer.test.ts create mode 100644 src/commands/developer.ts create mode 100644 src/types/developer.ts diff --git a/README.md b/README.md index 47df8a9ee1..100ba8ef8d 100644 --- a/README.md +++ b/README.md @@ -352,6 +352,35 @@ firecrawl search "AI startups funding" --sources news --tbs qdr:w --limit 15 --- +### `developer` - Search developer sources + +Search an index built for coding agents: GitHub issues, merged pull requests, repository READMEs, and curated documentation sites. Use it for a programming question: code behaviour, a library or framework, an API contract, an error message, or a known bug. + +```bash +firecrawl developer "axum middleware ordering" +``` + +#### Options + +| Option | Description | +| --------------------- | ----------------------------------------- | +| `--limit ` | Number of results (default: 20, max: 100) | +| `-o, --output ` | Save to file | +| `--json` | Output as compact JSON | +| `--pretty` | Pretty print JSON output | + +#### Examples + +```bash +# Investigate a known bug +firecrawl developer "tokio spawn_blocking panics thread limit" --limit 10 + +# Keep the full passages for an agent +firecrawl developer "tokio select cancellation safety" --json -o results.json +``` + +--- + ### `feedback` - Send endpoint job feedback Send concise feedback for a completed v2 `search`, `scrape`, `parse`, or `map` diff --git a/skills/firecrawl-search/SKILL.md b/skills/firecrawl-search/SKILL.md index 32bed16bd1..c4dfee04d3 100644 --- a/skills/firecrawl-search/SKILL.md +++ b/skills/firecrawl-search/SKILL.md @@ -45,6 +45,18 @@ holds `url`, `title`, and `description`, where `description` is the matched passage. Read the passages with `jq -r '.data.developer[] | .url, .description' .firecrawl/developer.json`. +The dedicated `firecrawl developer` command searches only that index and keeps +the full matched passages: + +```bash +# Developer search only, with full passages +firecrawl developer "your query" --limit 10 -o .firecrawl/developer.json --json +``` + +Each result holds `id`, `type` (`issue`, `pull_request`, `readme`, `doc`), +`url`, `title`, and `passages`. Read them with +`jq -r '.results[] | .url, .passages[].text' .firecrawl/developer.json`. + ## Options | Option | Description | diff --git a/src/__tests__/cli-argv.test.ts b/src/__tests__/cli-argv.test.ts index c6a347fda1..511f7fe795 100644 --- a/src/__tests__/cli-argv.test.ts +++ b/src/__tests__/cli-argv.test.ts @@ -7,6 +7,32 @@ describe('CLI argv parsing', () => { const cliPath = resolve(process.cwd(), 'dist/index.js'); const testWithBuiltCli = existsSync(cliPath) ? it : it.skip; + testWithBuiltCli('lists the developer command in root help output', () => { + const result = spawnSync(process.execPath, [cliPath, '--help'], { + cwd: process.cwd(), + encoding: 'utf8', + }); + + expect(result.status).toBe(0); + expect(result.stdout).toMatch(/^\s*developer\b/m); + }); + + testWithBuiltCli('parses the developer command and shows its help', () => { + const result = spawnSync( + process.execPath, + [cliPath, 'developer', '--help'], + { + cwd: process.cwd(), + encoding: 'utf8', + } + ); + + expect(result.status).toBe(0); + expect(result.stdout).toContain('Usage: firecrawl developer'); + expect(result.stdout).toContain('--limit'); + expect(result.stderr).not.toContain('unknown command'); + }); + testWithBuiltCli( 'parses subcommands when a wrapper leaves the entry script path in argv', () => { diff --git a/src/__tests__/commands/developer.test.ts b/src/__tests__/commands/developer.test.ts new file mode 100644 index 0000000000..8612a01635 --- /dev/null +++ b/src/__tests__/commands/developer.test.ts @@ -0,0 +1,177 @@ +/** + * Tests for developer command + */ + +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; +import { handleDeveloperSearchCommand } from '../../commands/developer'; +import { getClient } from '../../utils/client'; +import { initializeConfig } from '../../utils/config'; +import { writeOutput } from '../../utils/output'; +import { setupTest, teardownTest } from '../utils/mock-client'; + +vi.mock('../../utils/output', () => ({ writeOutput: vi.fn() })); + +vi.mock('../../utils/client', async () => { + const actual = await vi.importActual('../../utils/client'); + return { + ...actual, + getClient: vi.fn(), + }; +}); + +describe('handleDeveloperSearchCommand', () => { + let mockHttpGet: ReturnType; + + // Wrap a payload in the axios envelope returned by `client.http.get`. + // Mirrors the `/v2/developer/search` response shape: + // { success, results: [{ id, type, url, title, passages: [{ text }] }] } + const mockDeveloperResponse = (results: any[]) => ({ + data: { success: true, results }, + }); + + const sampleResult = { + id: 'issue:tokio-rs/tokio#2309', + type: 'issue', + url: 'https://github.com/tokio-rs/tokio/issues/2309', + title: 'spawn_blocking panics when exceeding the thread limit', + passages: [{ text: 'It will panic if this limit is too low.' }], + }; + + beforeEach(() => { + setupTest(); + initializeConfig({ + apiKey: 'test-api-key', + apiUrl: 'https://api.firecrawl.dev', + }); + + mockHttpGet = vi.fn(); + vi.mocked(getClient).mockReturnValue({ + http: { get: mockHttpGet }, + } as any); + }); + + afterEach(() => { + teardownTest(); + vi.clearAllMocks(); + }); + + describe('API call generation', () => { + it('calls /v2/developer/search with the query and integration tag', async () => { + mockHttpGet.mockResolvedValue(mockDeveloperResponse([sampleResult])); + + await handleDeveloperSearchCommand({ query: 'tokio spawn_blocking' }); + + expect(mockHttpGet).toHaveBeenCalledTimes(1); + expect(mockHttpGet).toHaveBeenCalledWith( + '/v2/developer/search?query=tokio+spawn_blocking&integration=cli' + ); + }); + + it('passes k when a limit is provided', async () => { + mockHttpGet.mockResolvedValue(mockDeveloperResponse([sampleResult])); + + await handleDeveloperSearchCommand({ + query: 'tokio spawn_blocking', + k: 5, + }); + + expect(mockHttpGet).toHaveBeenCalledWith( + '/v2/developer/search?query=tokio+spawn_blocking&k=5&integration=cli' + ); + }); + + it('passes apiUrl and apiKey to getClient when provided', async () => { + mockHttpGet.mockResolvedValue(mockDeveloperResponse([])); + + await handleDeveloperSearchCommand({ + query: 'test', + apiKey: 'other-key', + apiUrl: 'http://localhost:3002', + }); + + expect(getClient).toHaveBeenCalledWith({ + apiKey: 'other-key', + apiUrl: 'http://localhost:3002', + }); + }); + }); + + describe('output', () => { + it('renders id, type, title, url, and passage in readable output', async () => { + mockHttpGet.mockResolvedValue(mockDeveloperResponse([sampleResult])); + + await handleDeveloperSearchCommand({ query: 'tokio spawn_blocking' }); + + const [content] = vi.mocked(writeOutput).mock.calls[0]; + expect(content).toContain( + '## [issue:tokio-rs/tokio#2309] (issue) spawn_blocking panics when exceeding the thread limit' + ); + expect(content).toContain( + 'https://github.com/tokio-rs/tokio/issues/2309' + ); + expect(content).toContain('It will panic if this limit is too low.'); + }); + + it('joins multiple passages and clips long content', async () => { + mockHttpGet.mockResolvedValue( + mockDeveloperResponse([ + { + ...sampleResult, + passages: [{ text: 'first passage' }, { text: 'x'.repeat(5000) }], + }, + ]) + ); + + await handleDeveloperSearchCommand({ query: 'tokio spawn_blocking' }); + + const [content] = vi.mocked(writeOutput).mock.calls[0] as [string]; + expect(content).toContain('first passage\n---\nx'); + const body = content.split('\n').slice(2).join('\n'); + expect(body.length).toBeLessThanOrEqual(1200); + }); + + it('prints a placeholder when there are no results', async () => { + mockHttpGet.mockResolvedValue(mockDeveloperResponse([])); + + await handleDeveloperSearchCommand({ query: 'no hits' }); + + const [content] = vi.mocked(writeOutput).mock.calls[0]; + expect(content).toBe('(no results)'); + }); + + it('outputs the full envelope as JSON with --json', async () => { + mockHttpGet.mockResolvedValue(mockDeveloperResponse([sampleResult])); + + await handleDeveloperSearchCommand({ + query: 'tokio spawn_blocking', + json: true, + }); + + const [content] = vi.mocked(writeOutput).mock.calls[0] as [string]; + const parsed = JSON.parse(content); + expect(parsed.results[0].passages[0].text).toBe( + 'It will panic if this limit is too low.' + ); + }); + }); + + describe('error handling', () => { + it('exits with code 1 when the request fails', async () => { + mockHttpGet.mockRejectedValue(new Error('boom')); + const exitSpy = vi + .spyOn(process, 'exit') + .mockImplementation((() => undefined) as any); + const errorSpy = vi + .spyOn(console, 'error') + .mockImplementation(() => undefined); + + await handleDeveloperSearchCommand({ query: 'test' }); + + expect(errorSpy).toHaveBeenCalledWith('Error:', 'boom'); + expect(exitSpy).toHaveBeenCalledWith(1); + + exitSpy.mockRestore(); + errorSpy.mockRestore(); + }); + }); +}); diff --git a/src/commands/developer.ts b/src/commands/developer.ts new file mode 100644 index 0000000000..abfa80a2f6 --- /dev/null +++ b/src/commands/developer.ts @@ -0,0 +1,80 @@ +import { getClient, isKeylessMode, keylessGet } from '../utils/client'; +import { writeOutput } from '../utils/output'; +import type { DeveloperItem, DeveloperSearchOptions } from '../types/developer'; + +const BASE = '/v2/developer/search'; +const MAX_PASSAGE_CHARS = 1200; + +async function getDeveloper( + path: string, + options: DeveloperSearchOptions +): Promise { + const url = `${path}${path.includes('?') ? '&' : '?'}integration=cli`; + + if (isKeylessMode(options.apiKey, options.apiUrl)) { + return (await keylessGet(url)) as T; + } + + const app = getClient({ apiKey: options.apiKey, apiUrl: options.apiUrl }); + const response = await (app as any).http.get(url); + return (response?.data ?? {}) as T; +} + +function fmtDeveloper(results?: DeveloperItem[]): string { + if (!results || results.length === 0) return '(no results)'; + + return results + .map((item) => { + const kind = item.type ? ` (${item.type})` : ''; + const lines = [ + `## [${item.id ?? '?'}]${kind} ${item.title ?? '(untitled)'}`, + ]; + if (item.url) lines.push(item.url); + const body = (item.passages ?? []) + .map((passage) => passage.text ?? '') + .join('\n---\n') + .trim(); + lines.push(body ? body.slice(0, MAX_PASSAGE_CHARS) : '(no content)'); + return lines.join('\n'); + }) + .join('\n\n'); +} + +function writeDeveloperOutput( + data: unknown, + readable: string, + options: DeveloperSearchOptions +): void { + const content = + options.json || options.pretty + ? options.pretty + ? JSON.stringify(data, null, 2) + : JSON.stringify(data) + : readable; + writeOutput(content, options.output, !!options.output); +} + +function handleError(error: unknown): never { + console.error( + 'Error:', + error instanceof Error ? error.message : 'Unknown error occurred' + ); + process.exit(1); +} + +export async function handleDeveloperSearchCommand( + options: DeveloperSearchOptions +): Promise { + try { + const params = new URLSearchParams(); + params.append('query', options.query); + if (options.k != null) params.append('k', String(options.k)); + const data = await getDeveloper<{ results?: DeveloperItem[] }>( + `${BASE}?${params.toString()}`, + options + ); + writeDeveloperOutput(data, fmtDeveloper(data.results), options); + } catch (error) { + handleError(error); + } +} diff --git a/src/index.ts b/src/index.ts index 9a47d0d57d..a7d12586bd 100644 --- a/src/index.ts +++ b/src/index.ts @@ -20,6 +20,7 @@ import { handleMapCommand } from './commands/map'; import { handleParseCommand } from './commands/parse'; import { createMonitorCommand } from './commands/monitor'; import { handleSearchCommand } from './commands/search'; +import { handleDeveloperSearchCommand } from './commands/developer'; import { handleInspectPaperCommand, handleReadPaperCommand, @@ -1047,6 +1048,52 @@ function createSearchCommand(): Command { return searchCmd; } +/** + * Create and configure the developer command + */ +function createDeveloperCommand(): Command { + const developerCmd = new Command('developer') + .description( + 'Search an index built for coding agents: GitHub issues, merged PRs, repository READMEs, and curated documentation sites. Use it for a programming question: code behaviour, a library or framework, an API contract, an error message, or a known bug. Returns ranked results with id, type, url, title, and the matched passages in markdown.' + ) + .argument('', 'Natural-language developer question or search phrase') + .option( + '--limit ', + 'Number of results to return (default: 20, max: 100)', + parseInt + ) + .addOption(new Option('--k ').argParser(parseInt).hideHelp()) + .option( + '-k, --api-key ', + 'Firecrawl API key (overrides global --api-key)' + ) + .option('--api-url ', 'API URL (overrides global --api-url)') + .option('-o, --output ', 'Output file path (default: stdout)') + .option('--json', 'Output as compact JSON', false) + .option('--pretty', 'Pretty print JSON output', false) + .addHelpText( + 'after', + ` +Examples: + $ firecrawl developer "axum middleware ordering" --limit 10 + $ firecrawl developer "tokio select cancellation safety" --json +` + ) + .action(async (query, options) => { + await handleDeveloperSearchCommand({ + query, + k: researchLimit(options), + apiKey: options.apiKey, + apiUrl: options.apiUrl, + output: options.output, + json: options.json, + pretty: options.pretty, + }); + }); + + return developerCmd; +} + /** * Create and configure the research command group */ @@ -2038,6 +2085,7 @@ program.addCommand(createMapCommand()); program.addCommand(createParseCommand()); program.addCommand(createMonitorCommand()); program.addCommand(createSearchCommand()); +program.addCommand(createDeveloperCommand()); program.addCommand(createResearchCommand()); program.addCommand(createFeedbackCommand()); program.addCommand(createSearchFeedbackCommand()); diff --git a/src/types/developer.ts b/src/types/developer.ts new file mode 100644 index 0000000000..0a3d8e8620 --- /dev/null +++ b/src/types/developer.ts @@ -0,0 +1,17 @@ +export interface DeveloperSearchOptions { + query: string; + k?: number; + apiKey?: string; + apiUrl?: string; + output?: string; + json?: boolean; + pretty?: boolean; +} + +export interface DeveloperItem { + id?: string; + type?: string; + url?: string; + title?: string; + passages?: { text?: string }[]; +}