diff --git a/netlify.toml b/netlify.toml index 7f5a151..192389c 100644 --- a/netlify.toml +++ b/netlify.toml @@ -4,6 +4,10 @@ [functions] node_bundler = "esbuild" + external_node_modules = [] + +[functions."scrape"] + timeout = 26 [[redirects]] from = "/api/*" diff --git a/netlify/functions/scrape.mjs b/netlify/functions/scrape.mjs index 9ec71b9..5bc12cb 100644 --- a/netlify/functions/scrape.mjs +++ b/netlify/functions/scrape.mjs @@ -168,6 +168,7 @@ export async function handler(event) { submissions.sort((a, b) => (b.score || 0) - (a.score || 0)); } + // Map all posts first (skip duplicates) const posts = []; let lastCreatedUtc = null; @@ -177,12 +178,20 @@ export async function handler(event) { const post = mapPost(raw); lastCreatedUtc = raw.created_utc; + posts.push(post); + } - if (includeComments && post.num_comments > 0) { - post.comments = await fetchComments(raw.id); + // Fetch comments in parallel batches of 5 to stay within timeout + if (includeComments) { + const PARALLEL = 5; + const postsWithComments = posts.filter((p) => p.num_comments > 0); + for (let i = 0; i < postsWithComments.length; i += PARALLEL) { + const batch = postsWithComments.slice(i, i + PARALLEL); + const results = await Promise.all(batch.map((p) => fetchComments(p.id))); + for (let j = 0; j < batch.length; j++) { + batch[j].comments = results[j]; + } } - - posts.push(post); } // For pagination: if we got a full batch, use the last post's created_utc as cursor diff --git a/web/app.js b/web/app.js index 90885f1..33bbcb8 100644 --- a/web/app.js +++ b/web/app.js @@ -223,7 +223,7 @@ scrapeBtn.addEventListener("click", async () => { const customKeywords = keywordsEnabled ? getCustomKeywords() : {}; const categories = Object.keys(customKeywords).length > 0 ? customKeywords : DEFAULT_KEYWORDS; - const batchSize = includeComments ? 25 : 100; + const batchSize = includeComments ? 10 : 100; const totalTarget = limit * sortModes.length; // UI state