Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 7 additions & 2 deletions server/worldmonitor/conflict/v1/get-humanitarian-summary.ts
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,11 @@ import type {
import { CHROME_UA } from '../../../_shared/constants';
import { cachedFetchJson } from '../../../_shared/redis';

/** Max records per HAPI conflict-events query */
const HAPI_QUERY_LIMIT = 1000;
/** Timeout for HAPI API requests */
const HAPI_TIMEOUT_MS = 15_000;

const REDIS_CACHE_KEY = 'conflict:humanitarian:v1';
const REDIS_CACHE_TTL = 21600; // 6 hr — monthly humanitarian data

Expand Down Expand Up @@ -44,7 +49,7 @@ interface HapiCountryAgg {
async function fetchHapiSummary(countryCode: string): Promise<HumanitarianCountrySummary | undefined> {
try {
const appId = btoa('worldmonitor:monitor@worldmonitor.app');
let url = `https://hapi.humdata.org/api/v2/coordination-context/conflict-events?output_format=json&limit=1000&offset=0&app_identifier=${appId}`;
let url = `https://hapi.humdata.org/api/v2/coordination-context/conflict-events?output_format=json&limit=${HAPI_QUERY_LIMIT}&offset=0&app_identifier=${appId}`;

// Filter by country — if a specific country was requested but has no ISO3 mapping,
// return undefined immediately rather than silently returning unrelated data (BLOCKING-1 fix)
Expand All @@ -56,7 +61,7 @@ async function fetchHapiSummary(countryCode: string): Promise<HumanitarianCountr

const response = await fetch(url, {
headers: { Accept: 'application/json', 'User-Agent': CHROME_UA },
signal: AbortSignal.timeout(15000),
signal: AbortSignal.timeout(HAPI_TIMEOUT_MS),
});

if (!response.ok) return undefined;
Expand Down
13 changes: 10 additions & 3 deletions server/worldmonitor/displacement/v1/get-displacement-summary.ts
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,13 @@ import type {
import { CHROME_UA } from '../../../_shared/constants';
import { cachedFetchJson } from '../../../_shared/redis';

/** Max items per UNHCR Population API page */
const UNHCR_PAGE_LIMIT = 10_000;
/** Safety cap on the number of pages to paginate through */
const MAX_PAGE_GUARD = 25;
/** Default number of refugee flow corridors to return */
const DEFAULT_FLOW_LIMIT = 50;

const REDIS_CACHE_KEY = 'displacement:summary:v1';
const REDIS_CACHE_TTL = 43200; // 12 hr — annual UNHCR data, very slow-moving

Expand Down Expand Up @@ -50,8 +57,8 @@ interface UnhcrRawItem {

/** Paginate through all UNHCR Population API pages for a given year. */
async function fetchUnhcrYearItems(year: number): Promise<UnhcrRawItem[] | null> {
const limit = 10000;
const maxPageGuard = 25;
const limit = UNHCR_PAGE_LIMIT;
const maxPageGuard = MAX_PAGE_GUARD;
const items: UnhcrRawItem[] = [];

for (let page = 1; page <= maxPageGuard; page++) {
Expand Down Expand Up @@ -330,7 +337,7 @@ export async function getDisplacementSummary(
if (req.countryLimit > 0) {
summary.countries = summary.countries.slice(0, req.countryLimit);
}
const flowLimit = req.flowLimit > 0 ? req.flowLimit : 50;
const flowLimit = req.flowLimit > 0 ? req.flowLimit : DEFAULT_FLOW_LIMIT;
summary.topFlows = summary.topFlows.slice(0, flowLimit);
return { summary };
}
Expand Down
9 changes: 7 additions & 2 deletions server/worldmonitor/economic/v1/get-energy-capacity.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,11 @@ import type {
import { CHROME_UA } from '../../../_shared/constants';
import { cachedFetchJson } from '../../../_shared/redis';

/** Max rows returned from EIA capability endpoint */
const EIA_MAX_RESULTS = 5000;
/** Timeout for EIA API requests */
const EIA_TIMEOUT_MS = 15_000;

const REDIS_CACHE_KEY = 'economic:capacity:v1';
const REDIS_CACHE_TTL = 86400; // 24h — annual data barely changes
const DEFAULT_YEARS = 20;
Expand Down Expand Up @@ -54,14 +59,14 @@ async function fetchCapacityForSource(
'facets[energysourceid][]': sourceCode,
'sort[0][column]': 'period',
'sort[0][direction]': 'desc',
length: '5000',
length: String(EIA_MAX_RESULTS),
start: String(startYear),
});

const url = `https://api.eia.gov/v2/electricity/state-electricity-profiles/capability/data/?${params}`;
const response = await fetch(url, {
headers: { Accept: 'application/json', 'User-Agent': CHROME_UA },
signal: AbortSignal.timeout(15000),
signal: AbortSignal.timeout(EIA_TIMEOUT_MS),
});

if (!response.ok) return new Map();
Expand Down
9 changes: 7 additions & 2 deletions server/worldmonitor/economic/v1/get-energy-prices.ts
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,11 @@ import type {
import { CHROME_UA } from '../../../_shared/constants';
import { cachedFetchJson } from '../../../_shared/redis';

/** Number of recent data points to fetch for price comparison (current + previous) */
const EIA_PRICE_HISTORY_LENGTH = 2;
/** Timeout for EIA API requests */
const EIA_PRICE_TIMEOUT_MS = 10_000;

const REDIS_CACHE_KEY = 'economic:energy:v1';
const REDIS_CACHE_TTL = 3600; // 1 hr — weekly EIA data

Expand Down Expand Up @@ -52,12 +57,12 @@ async function fetchEiaSeries(
'facets[series][]': config.seriesFacet,
'sort[0][column]': 'period',
'sort[0][direction]': 'desc',
length: '2',
length: String(EIA_PRICE_HISTORY_LENGTH),
});

const response = await fetch(`https://api.eia.gov${config.apiPath}?${params}`, {
headers: { Accept: 'application/json', 'User-Agent': CHROME_UA },
signal: AbortSignal.timeout(10000),
signal: AbortSignal.timeout(EIA_PRICE_TIMEOUT_MS),
});

if (!response.ok) return null;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,10 @@ let fallbackOutagesCache: { data: ListInternetOutagesResponse; ts: number } | nu
// ========================================================================

const CLOUDFLARE_RADAR_URL = 'https://api.cloudflare.com/client/v4/radar/annotations/outages';
/** Max outage annotations to request from Cloudflare Radar */
const OUTAGE_QUERY_LIMIT = 50;
/** Max ASN detail entries to include per outage */
const MAX_ASN_DETAILS = 2;

// ========================================================================
// Cloudflare Radar types
Expand Down Expand Up @@ -142,7 +146,7 @@ export async function listInternetOutages(
if (!token) return null;

const response = await fetch(
`${CLOUDFLARE_RADAR_URL}?dateRange=7d&limit=50`,
`${CLOUDFLARE_RADAR_URL}?dateRange=7d&limit=${OUTAGE_QUERY_LIMIT}`,
{
headers: { Authorization: `Bearer ${token}`, 'User-Agent': CHROME_UA },
signal: AbortSignal.timeout(UPSTREAM_TIMEOUT_MS),
Expand All @@ -168,7 +172,7 @@ export async function listInternetOutages(
const categories: string[] = ['Cloudflare Radar'];
if (raw.outage?.outageCause) categories.push(raw.outage.outageCause.replace(/_/g, ' '));
if (raw.outage?.outageType) categories.push(raw.outage.outageType);
for (const asn of raw.asnsDetails?.slice(0, 2) || []) {
for (const asn of raw.asnsDetails?.slice(0, MAX_ASN_DETAILS) || []) {
if (asn.name) categories.push(asn.name);
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,9 @@ import {
type BaselineEntry,
} from './_shared';

/** Max baseline updates processed per request */
const MAX_BATCH_UPDATES = 20;

// ========================================================================
// RPC implementation
// ========================================================================
Expand All @@ -28,7 +31,7 @@ export async function recordBaselineSnapshot(
return { updated: 0, error: 'Body must have updates array' };
}

const batch = updates.slice(0, 20);
const batch = updates.slice(0, MAX_BATCH_UPDATES);
const now = new Date();
const weekday = now.getUTCDay();
const month = now.getUTCMonth() + 1;
Expand Down
17 changes: 13 additions & 4 deletions server/worldmonitor/military/v1/get-aircraft-details-batch.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,15 @@ import { mapWingbitsDetails } from './_shared';
import { CHROME_UA } from '../../../_shared/constants';
import { getCachedJsonBatch, cachedFetchJson } from '../../../_shared/redis';

/** Max ICAO24 identifiers per batch request */
const MAX_BATCH_SIZE = 10;
/** Delay between sequential Wingbits API fetches to avoid rate-limiting */
const FETCH_DELAY_MS = 100;
/** Timeout for individual Wingbits API requests */
const WINGBITS_TIMEOUT_MS = 10_000;
/** Cache TTL for aircraft details (24 hours) */
const AIRCRAFT_CACHE_TTL = 24 * 60 * 60;

interface CachedAircraftDetails {
details: AircraftDetails | null;
configured: boolean;
Expand All @@ -26,11 +35,11 @@ export async function getAircraftDetailsBatch(
.map((id) => id.trim().toLowerCase())
.filter((id) => id.length > 0);
const uniqueSorted = Array.from(new Set(normalized)).sort();
const limitedList = uniqueSorted.slice(0, 10);
const limitedList = uniqueSorted.slice(0, MAX_BATCH_SIZE);

// Redis shared cache — batch GET all keys in a single pipeline round-trip
const SINGLE_KEY = 'military:aircraft:v1';
const SINGLE_TTL = 24 * 60 * 60;
const SINGLE_TTL = AIRCRAFT_CACHE_TTL;
const results: Record<string, AircraftDetails> = {};
const toFetch: string[] = [];

Expand Down Expand Up @@ -62,7 +71,7 @@ export async function getAircraftDetailsBatch(
try {
const resp = await fetch(`https://customer-api.wingbits.com/v1/flights/details/${icao24}`, {
headers: { 'x-api-key': apiKey, Accept: 'application/json', 'User-Agent': CHROME_UA },
signal: AbortSignal.timeout(10_000),
signal: AbortSignal.timeout(WINGBITS_TIMEOUT_MS),
});
if (resp.status === 404) {
return { details: null, configured: true };
Expand All @@ -77,7 +86,7 @@ export async function getAircraftDetailsBatch(
},
);
if (cacheResult?.details) results[icao24] = cacheResult.details;
if (i < toFetch.length - 1) await delay(100);
if (i < toFetch.length - 1) await delay(FETCH_DELAY_MS);
}

return {
Expand Down
17 changes: 13 additions & 4 deletions server/worldmonitor/positive-events/v1/list-positive-geo-events.ts
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,15 @@ import { markNoCacheResponse } from '../../../_shared/response-headers';

const GDELT_GEO_URL = 'https://api.gdeltproject.org/api/v2/geo/geo';

/** Delay between sequential GDELT queries to avoid rate-limiting */
const GDELT_QUERY_DELAY_MS = 500;
/** Max records per GDELT GEO API request */
const GDELT_MAX_RECORDS = 75;
/** Timeout for GDELT API requests */
const GDELT_TIMEOUT_MS = 10_000;
/** Minimum article count to include an event (noise filter) */
const MIN_EVENT_COUNT = 3;

const REDIS_CACHE_KEY = 'positive-events:geo:v1';
const REDIS_CACHE_TTL = 900;

Expand All @@ -31,12 +40,12 @@ async function fetchGdeltGeoPositive(query: string): Promise<PositiveGeoEvent[]>
query,
format: 'geojson',
timespan: '24h',
maxrecords: '75',
maxrecords: String(GDELT_MAX_RECORDS),
});

const response = await fetch(`${GDELT_GEO_URL}?${params}`, {
headers: { Accept: 'application/json' },
signal: AbortSignal.timeout(10000),
signal: AbortSignal.timeout(GDELT_TIMEOUT_MS),
});

if (!response.ok) return [];
Expand All @@ -53,7 +62,7 @@ async function fetchGdeltGeoPositive(query: string): Promise<PositiveGeoEvent[]>
if (name.startsWith('ERROR:') || name.includes('unknown error')) continue;

const count: number = feature.properties?.count || 1;
if (count < 3) continue; // Noise filter
if (count < MIN_EVENT_COUNT) continue; // Noise filter

const coords = feature.geometry?.coordinates;
if (!Array.isArray(coords) || coords.length < 2) continue;
Expand Down Expand Up @@ -97,7 +106,7 @@ export async function listPositiveGeoEvents(

for (let i = 0; i < POSITIVE_QUERIES.length; i++) {
if (i > 0) {
await new Promise(r => setTimeout(r, 500));
await new Promise(r => setTimeout(r, GDELT_QUERY_DELAY_MS));
}

try {
Expand Down
14 changes: 10 additions & 4 deletions server/worldmonitor/research/v1/list-hackernews-items.ts
Original file line number Diff line number Diff line change
Expand Up @@ -22,18 +22,24 @@ const REDIS_CACHE_TTL = 600; // 10 min

const ALLOWED_HN_FEEDS = new Set(['top', 'new', 'best', 'ask', 'show', 'job']);
const HN_MAX_CONCURRENCY = 10;
/** Default number of HN stories per page */
const DEFAULT_PAGE_SIZE = 30;
/** Timeout for fetching story IDs list */
const HN_IDS_TIMEOUT_MS = 10_000;
/** Timeout for fetching individual story items */
const HN_ITEM_TIMEOUT_MS = 5_000;

// ---------- Fetch ----------

async function fetchHackernewsItems(req: ListHackernewsItemsRequest): Promise<HackernewsItem[]> {
const feedType = ALLOWED_HN_FEEDS.has(req.feedType) ? req.feedType : 'top';
const pageSize = req.pageSize || 30;
const pageSize = req.pageSize || DEFAULT_PAGE_SIZE;

// Step 1: Fetch story IDs
const idsUrl = `https://hacker-news.firebaseio.com/v0/${feedType}stories.json`;
const idsResponse = await fetch(idsUrl, {
headers: { 'User-Agent': CHROME_UA },
signal: AbortSignal.timeout(10000),
signal: AbortSignal.timeout(HN_IDS_TIMEOUT_MS),
});

if (!idsResponse.ok) return [];
Expand All @@ -52,7 +58,7 @@ async function fetchHackernewsItems(req: ListHackernewsItemsRequest): Promise<Ha
try {
const res = await fetch(
`https://hacker-news.firebaseio.com/v0/item/${id}.json`,
{ headers: { 'User-Agent': CHROME_UA }, signal: AbortSignal.timeout(5000) },
{ headers: { 'User-Agent': CHROME_UA }, signal: AbortSignal.timeout(HN_ITEM_TIMEOUT_MS) },
);
if (!res.ok) return null;
const raw: any = await res.json();
Expand Down Expand Up @@ -87,7 +93,7 @@ export async function listHackernewsItems(
): Promise<ListHackernewsItemsResponse> {
try {
const feedType = ALLOWED_HN_FEEDS.has(req.feedType) ? req.feedType : 'top';
const cacheKey = `${REDIS_CACHE_KEY}:${feedType}:${req.pageSize || 30}`;
const cacheKey = `${REDIS_CACHE_KEY}:${feedType}:${req.pageSize || DEFAULT_PAGE_SIZE}`;
const result = await cachedFetchJson<ListHackernewsItemsResponse>(cacheKey, REDIS_CACHE_TTL, async () => {
const items = await fetchHackernewsItems(req);
return items.length > 0 ? { items, pagination: undefined } : null;
Expand Down
17 changes: 13 additions & 4 deletions server/worldmonitor/unrest/v1/list-unrest-events.ts
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,15 @@ import { CHROME_UA } from '../../../_shared/constants';
import { cachedFetchJson } from '../../../_shared/redis';
import { fetchAcledCached } from '../../../_shared/acled';

/** Max records per GDELT GEO API request for unrest events */
const GDELT_MAX_RECORDS = 250;
/** Timeout for GDELT API requests */
const GDELT_TIMEOUT_MS = 10_000;
/** Minimum report count to include a GDELT event (noise filter) */
const MIN_REPORT_COUNT = 5;
/** Report count threshold above which GDELT confidence is HIGH */
const HIGH_CONFIDENCE_THRESHOLD = 20;

const REDIS_CACHE_KEY = 'unrest:events:v1';
const REDIS_CACHE_TTL = 900; // 15 min — ACLED + GDELT merge

Expand Down Expand Up @@ -87,13 +96,13 @@ async function fetchGdeltEvents(): Promise<UnrestEvent[]> {
const params = new URLSearchParams({
query: 'protest',
format: 'geojson',
maxrecords: '250',
maxrecords: String(GDELT_MAX_RECORDS),
timespan: '7d',
});

const response = await fetch(`${GDELT_GEO_URL}?${params}`, {
headers: { Accept: 'application/json', 'User-Agent': CHROME_UA },
signal: AbortSignal.timeout(10000),
signal: AbortSignal.timeout(GDELT_TIMEOUT_MS),
});

if (!response.ok) return [];
Expand All @@ -108,7 +117,7 @@ async function fetchGdeltEvents(): Promise<UnrestEvent[]> {
if (!name || seenLocations.has(name)) continue;

const count: number = feature.properties?.count || 1;
if (count < 5) continue; // Filter noise
if (count < MIN_REPORT_COUNT) continue; // Filter noise

const coords = feature.geometry?.coordinates;
if (!Array.isArray(coords) || coords.length < 2) continue;
Expand Down Expand Up @@ -143,7 +152,7 @@ async function fetchGdeltEvents(): Promise<UnrestEvent[]> {
sourceType: 'UNREST_SOURCE_TYPE_GDELT' as UnrestSourceType,
tags: [],
actors: [],
confidence: (count > 20
confidence: (count > HIGH_CONFIDENCE_THRESHOLD
? 'CONFIDENCE_LEVEL_HIGH'
: 'CONFIDENCE_LEVEL_MEDIUM') as ConfidenceLevel,
});
Expand Down