diff --git a/services/bin/shodan-internetdb.js b/services/bin/shodan-internetdb.js new file mode 100644 index 00000000..1d2cac0c --- /dev/null +++ b/services/bin/shodan-internetdb.js @@ -0,0 +1,5 @@ +#!/usr/bin/env node +import { fileURLToPath } from "node:url"; +import { runServiceMain } from "@chaitin-ai/octobus-sdk"; +import { service } from "../shodan__internetdb/src/service.js"; +runServiceMain(service, { entryFile: fileURLToPath(new URL("../shodan__internetdb/bin/shodan-internetdb.js", import.meta.url)) }); diff --git a/services/shodan__internetdb/README.md b/services/shodan__internetdb/README.md new file mode 100644 index 00000000..8f7c3771 --- /dev/null +++ b/services/shodan__internetdb/README.md @@ -0,0 +1,41 @@ +# Shodan InternetDB Service Package + +OctoBus package for [Shodan InternetDB](https://internetdb.shodan.io/) — a free, no-signup IP intelligence API. + +## Features + +- **No API key required** — unlimited usage +- **No signup needed** +- **Shodan** is a well-known security company (network device search engine) + +## Methods + +| RPC | HTTP API | Type | Description | +|-----|----------|------|-------------| +| `LookupIP` | `GET /{ip}` | Read | IP intelligence (ports, hostnames, CVEs, tags) | + +## Configuration + +| Config field | Default | Description | +|-------------|---------|-------------| +| `timeoutMs` | 10000 | HTTP request timeout | + +No authentication required — `secret.schema.json` is empty. + +## Usage + +```bash +# Create instance +octobus instance create shodan-test \ + --service shodan-internetdb + +# Create capset +octobus capset create threat-intel --name Threat-Intel +octobus capset add-instance threat-intel shodan-test + +# Lookup IP +curl -X POST \ + 'http://127.0.0.1:9000/capsets/threat-intel/connect/shodan-test/Shodan_InternetDB.Shodan_InternetDB/LookupIP' \ + -H 'Content-Type: application/json' \ + -d '{"ip":"8.8.8.8"}' +``` diff --git a/services/shodan__internetdb/bin/shodan-internetdb.js b/services/shodan__internetdb/bin/shodan-internetdb.js new file mode 100644 index 00000000..604f5241 --- /dev/null +++ b/services/shodan__internetdb/bin/shodan-internetdb.js @@ -0,0 +1,4 @@ +#!/usr/bin/env node +import { runServiceMain } from '@chaitin-ai/octobus-sdk'; +import { service } from '../src/service.js'; +runServiceMain(service); diff --git a/services/shodan__internetdb/config.schema.json b/services/shodan__internetdb/config.schema.json new file mode 100644 index 00000000..c1bc348a --- /dev/null +++ b/services/shodan__internetdb/config.schema.json @@ -0,0 +1,13 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "additionalProperties": true, + "properties": { + "timeoutMs": { + "type": "integer", + "minimum": 1, + "default": 10000, + "description": "HTTP timeout in milliseconds." + } + } +} diff --git a/services/shodan__internetdb/package.json b/services/shodan__internetdb/package.json new file mode 100644 index 00000000..e8b472f4 --- /dev/null +++ b/services/shodan__internetdb/package.json @@ -0,0 +1,12 @@ +{ + "name": "shodan-internetdb", + "version": "0.0.0", + "private": true, + "type": "module", + "bin": { + "shodan-internetdb": "bin/shodan-internetdb.js" + }, + "dependencies": { + "@chaitin-ai/octobus-sdk": "^0.5.0" + } +} diff --git a/services/shodan__internetdb/proto/shodan_internetdb.proto b/services/shodan__internetdb/proto/shodan_internetdb.proto new file mode 100644 index 00000000..781ef9d9 --- /dev/null +++ b/services/shodan__internetdb/proto/shodan_internetdb.proto @@ -0,0 +1,24 @@ +syntax = "proto3"; + +package Shodan_InternetDB; + +option go_package = "miner/grpc-service/Shodan_InternetDB"; + +service Shodan_InternetDB { + rpc LookupIP(LookupIPRequest) returns (LookupIPResponse) {} +} + +message LookupIPRequest { + string ip = 1; +} + +message LookupIPResponse { + int32 code = 1; + string message = 2; + string ip = 3; + repeated string hostnames = 4; + repeated int32 ports = 5; + repeated string cpes = 6; + repeated string tags = 7; + repeated string vulns = 8; +} diff --git a/services/shodan__internetdb/secret.schema.json b/services/shodan__internetdb/secret.schema.json new file mode 100644 index 00000000..16e4463a --- /dev/null +++ b/services/shodan__internetdb/secret.schema.json @@ -0,0 +1,6 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "additionalProperties": true, + "properties": {} +} diff --git a/services/shodan__internetdb/service.json b/services/shodan__internetdb/service.json new file mode 100644 index 00000000..727bc510 --- /dev/null +++ b/services/shodan__internetdb/service.json @@ -0,0 +1,29 @@ +{ + "schema": "chaitin.octobus.service.v1", + "name": "shodan-internetdb", + "displayName": "Shodan InternetDB", + "description": "OctoBus package for Shodan InternetDB — free IP intelligence (ports, hostnames, CVEs) without API key.", + "runtime": { + "mode": "long-running" + }, + "proto": { + "roots": [ + "proto" + ], + "files": [ + "proto/shodan_internetdb.proto" + ] + }, + "configSchema": "config.schema.json", + "secretSchema": "secret.schema.json", + "sdk": { + "cli": { + "commands": { + "Shodan_InternetDB.Shodan_InternetDB/LookupIP": { + "name": "lookup-ip", + "description": "Look up IP intelligence from Shodan InternetDB (ports, hostnames, CVEs)." + } + } + } + } +} diff --git a/services/shodan__internetdb/src/service.js b/services/shodan__internetdb/src/service.js new file mode 100644 index 00000000..be0cfa9c --- /dev/null +++ b/services/shodan__internetdb/src/service.js @@ -0,0 +1,4 @@ +import { defineService } from '@chaitin-ai/octobus-sdk'; +import { handlers } from './shodan-internetdb.js'; +export { handlers } from './shodan-internetdb.js'; +export const service = defineService({ handlers }); diff --git a/services/shodan__internetdb/src/shodan-internetdb.js b/services/shodan__internetdb/src/shodan-internetdb.js new file mode 100644 index 00000000..5b399dc0 --- /dev/null +++ b/services/shodan__internetdb/src/shodan-internetdb.js @@ -0,0 +1,154 @@ +import net from 'node:net'; +import { GrpcError, grpcStatus } from '@chaitin-ai/octobus-sdk'; + +// ---- Method paths ---- + +const METHOD_LOOKUP_FULL = 'Shodan_InternetDB.Shodan_InternetDB/LookupIP'; + +// ---- Constants ---- + +const API_BASE = 'https://internetdb.shodan.io'; +const SDK_REF = 'Shodan_InternetDB'; +const DEFAULT_TIMEOUT_MS = 10000; + +// ---- Error helpers ---- + +const grpcCodeFor = (code) => ({ + FAILED_PRECONDITION: grpcStatus.FAILED_PRECONDITION, + INVALID_ARGUMENT: grpcStatus.INVALID_ARGUMENT, + PERMISSION_DENIED: grpcStatus.PERMISSION_DENIED, + UNAVAILABLE: grpcStatus.UNAVAILABLE, + UNKNOWN: grpcStatus.UNKNOWN, +})[code] ?? grpcStatus.UNKNOWN; + +const errorWithCode = (code, message) => { + const err = new GrpcError(grpcCodeFor(code), `${code}: ${message}`); + err.legacyCode = code; + return err; +}; + +// ---- Value helpers ---- + +const hasOwn = (obj, key) => Object.prototype.hasOwnProperty.call(obj ?? {}, key); +const firstDefined = (...values) => values.find((v) => v !== undefined && v !== null); + +const unwrapString = (source) => { + if (source === undefined || source === null) return ''; + if (typeof source === 'object' && source !== null && hasOwn(source, 'value')) return unwrapString(source.value); + return String(source); +}; + +const logInfo = (meta, action, payload) => { + const prefix = `[${SDK_REF}][${action}]`; + try { console.log(prefix, JSON.stringify(payload)); } catch { console.log(prefix, payload); } +}; + +const logError = (meta, action, payload) => { + const prefix = `[${SDK_REF}][${action}]`; + try { console.error(prefix, JSON.stringify(payload)); } catch { console.error(prefix, payload); } +}; + +// ---- HTTP ---- + +const parseJson = (text) => { + if (!String(text || '').trim()) return null; + try { return JSON.parse(text); } catch { throw errorWithCode('UNKNOWN', 'response is not valid JSON'); } +}; + +const mapHttpError = (res, bodyText) => { + if (res.status === 401 || res.status === 403) throw errorWithCode('PERMISSION_DENIED', `upstream http ${res.status}`); + if (res.status === 429) throw errorWithCode('UNAVAILABLE', `upstream http ${res.status}`); + if (res.status >= 400 && res.status < 500) throw errorWithCode('FAILED_PRECONDITION', `upstream http ${res.status}`); + throw errorWithCode('UNAVAILABLE', `upstream http ${res.status}`); +}; + +const fetchJson = async (url, init, { bindings = {}, timeoutMs }) => { + let res; + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), timeoutMs || DEFAULT_TIMEOUT_MS); + try { + res = await fetch(url, { ...init, signal: controller.signal }); + const text = await res.text(); + if (!res.ok) mapHttpError(res, text); + return { json: parseJson(text), text }; + } catch (err) { + if (err instanceof GrpcError) throw err; + const reason = err?.cause?.message || err?.message || 'fetch failed'; + throw errorWithCode('UNAVAILABLE', reason); + } finally { + clearTimeout(timer); + } +}; + +// ---- Context resolution ---- + +const mergedBindings = (ctx = {}) => ({ ...(ctx.config ?? {}), ...(ctx.secret ?? {}), ...(ctx.bindings ?? {}) }); + +const resolveCallContext = (ctx = {}) => ({ + ...ctx, bindings: mergedBindings(ctx), limits: ctx.limits ?? {}, meta: ctx.meta ?? {}, req: ctx.req ?? ctx.request ?? {}, +}); + +const resolveTimeoutMs = (ctx = {}, bindings = {}) => + firstDefined(ctx.limits?.timeoutMs, bindings.timeoutMs, DEFAULT_TIMEOUT_MS); + +// ---- Handlers ---- + +const makeRuntime = (ctx = {}) => { + const callCtx = resolveCallContext(ctx); + const bindings = callCtx.bindings || {}; + const meta = callCtx.meta || {}; + const timeoutMs = resolveTimeoutMs(callCtx, bindings); + + const runLookup = async (req = {}) => { + const ip = unwrapString(firstDefined(req.ip)).trim(); + if (!ip) throw errorWithCode('INVALID_ARGUMENT', 'ip is required'); + if (!net.isIP(ip)) throw errorWithCode('INVALID_ARGUMENT', 'invalid IP address format'); + + const url = `${API_BASE}/${encodeURIComponent(ip)}`; + + logInfo(meta, 'LookupIP:start', { ip }); + + let result; + try { + result = await fetchJson(url, { method: 'GET' }, { bindings, timeoutMs }); + } catch (err) { + logError(meta, 'LookupIP:http-error', { ip, error: err.message }); + throw err; + } + + logInfo(meta, 'LookupIP:success', { ip }); + + return { + code: 0, + message: 'ok', + ip: result.json?.ip || ip, + hostnames: result.json?.hostnames || [], + ports: result.json?.ports || [], + cpes: result.json?.cpes || [], + tags: result.json?.tags || [], + vulns: result.json?.vulns || [], + }; + }; + + return { runLookup }; +}; + +// ---- Exports ---- + +export const handlers = { + [METHOD_LOOKUP_FULL]: (ctx) => makeRuntime(ctx).runLookup(ctx.request ?? {}), +}; + +export const _test = { + errorWithCode, + firstDefined, + hasOwn, + logInfo, + logError, + makeRuntime, + mapHttpError, + parseJson, + resolveCallContext, + resolveTimeoutMs, + unwrapString, +}; diff --git a/services/shodan__internetdb/test/shodan-internetdb.test.js b/services/shodan__internetdb/test/shodan-internetdb.test.js new file mode 100644 index 00000000..8adfa16f --- /dev/null +++ b/services/shodan__internetdb/test/shodan-internetdb.test.js @@ -0,0 +1,125 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; + +import { GrpcError } from '@chaitin-ai/octobus-sdk'; + +import { + handlers, + _test, +} from '../src/shodan-internetdb.js'; +import { service } from '../src/service.js'; + +const originalFetch = globalThis.fetch; +const originalConsoleLog = console.log; +const originalConsoleError = console.error; + +const LOOKUP_RESP = { + ip: '8.8.8.8', + hostnames: ['dns.google'], + ports: [53, 443], + cpes: ['cpe:/a:google:dns'], + tags: ['cdn'], + vulns: ['CVE-2023-1234'], +}; + +const ctx = (overrides = {}) => ({ + config: { ...(overrides.config || {}) }, + secret: { ...(overrides.secret || {}) }, + bindings: { ...(overrides.bindings || {}) }, + limits: { timeoutMs: 2000, ...(overrides.limits || {}) }, + meta: { instance_id: 'inst', request_id: 'req', ...(overrides.meta || {}) }, +}); + +const response = (status, body) => ({ + ok: status >= 200 && status < 300, + status, + headers: { get: () => 'application/json' }, + text: async () => (typeof body === 'string' ? body : JSON.stringify(body)), +}); + +const setFetch = (impl) => { globalThis.fetch = impl; }; + +const expectGrpcError = async (fn, legacyCode) => { + let caught; + try { await fn(); } catch (err) { caught = err; } + assert.ok(caught, 'expected function to reject'); + assert.ok(caught instanceof GrpcError); + assert.equal(caught.legacyCode, legacyCode); + assert.match(caught.message, new RegExp(`^${legacyCode}:`)); +}; + +test.beforeEach(() => { console.log = () => {}; console.error = () => {}; }); +test.afterEach(() => { + globalThis.fetch = originalFetch; + console.log = originalConsoleLog; + console.error = originalConsoleError; +}); + +test('service exports handlers', () => { + assert.equal(typeof service, 'object'); + assert.equal(typeof handlers['Shodan_InternetDB.Shodan_InternetDB/LookupIP'], 'function'); +}); + +test('LookupIP returns IP intelligence data', async () => { + setFetch(async (url, init) => { + assert.ok(url.startsWith('https://internetdb.shodan.io/8.8.8.8')); + assert.equal(init.method, 'GET'); + return response(200, LOOKUP_RESP); + }); + + const result = await handlers['Shodan_InternetDB.Shodan_InternetDB/LookupIP']({ ...ctx(), request: { ip: '8.8.8.8' } }); + assert.equal(result.code, 0); + assert.equal(result.ip, '8.8.8.8'); + assert.ok(result.hostnames.includes('dns.google')); + assert.ok(result.ports.includes(53)); + assert.ok(result.cpes.includes('cpe:/a:google:dns')); + assert.ok(result.tags.includes('cdn')); + assert.ok(result.vulns.includes('CVE-2023-1234')); +}); + +test('LookupIP validates ip', async () => { + await expectGrpcError( + () => handlers['Shodan_InternetDB.Shodan_InternetDB/LookupIP']({ ...ctx(), request: {} }), + 'INVALID_ARGUMENT', + ); +}); + +test('handles empty response', async () => { + setFetch(async () => ({ ok: true, status: 200, headers: { get: () => 'text/plain' }, text: async () => '' })); + const result = await handlers['Shodan_InternetDB.Shodan_InternetDB/LookupIP']({ ...ctx(), request: { ip: '1.1.1.1' } }); + assert.equal(result.code, 0); + assert.equal(result.ip, '1.1.1.1'); +}); + +test('maps HTTP errors', async () => { + setFetch(async () => response(401, 'unauthorized')); + await expectGrpcError( + () => handlers['Shodan_InternetDB.Shodan_InternetDB/LookupIP']({ ...ctx(), request: { ip: '0.0.0.0' } }), + 'PERMISSION_DENIED', + ); + setFetch(async () => response(403, 'forbidden')); + await expectGrpcError( + () => handlers['Shodan_InternetDB.Shodan_InternetDB/LookupIP']({ ...ctx(), request: { ip: '0.0.0.0' } }), + 'PERMISSION_DENIED', + ); + setFetch(async () => response(404, 'not found')); + await expectGrpcError( + () => handlers['Shodan_InternetDB.Shodan_InternetDB/LookupIP']({ ...ctx(), request: { ip: '0.0.0.0' } }), + 'FAILED_PRECONDITION', + ); + setFetch(async () => response(429, 'too many')); + await expectGrpcError( + () => handlers['Shodan_InternetDB.Shodan_InternetDB/LookupIP']({ ...ctx(), request: { ip: '0.0.0.0' } }), + 'UNAVAILABLE', + ); + setFetch(async () => response(500, 'error')); + await expectGrpcError( + () => handlers['Shodan_InternetDB.Shodan_InternetDB/LookupIP']({ ...ctx(), request: { ip: '0.0.0.0' } }), + 'UNAVAILABLE', + ); +}); + +test('helper utilities', () => { + assert.equal(_test.firstDefined(undefined, null, 'x'), 'x'); + assert.equal(_test.unwrapString({ value: { value: 'nested' } }), 'nested'); +});