From 8c1413b10654eb4fb04dafd35c53a774efc8d8a7 Mon Sep 17 00:00:00 2001 From: mikim <1441941+mikim@users.noreply.github.com> Date: Thu, 6 Aug 2026 13:11:09 +0900 Subject: [PATCH] Fix robots.txt leaking /api to every named bot, and 9h-off cron MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit robots.txt Every named bot had a bare `{ userAgent: "X", allow: "/" }` group. robots.txt group matching is exclusive — a crawler obeys only the most specific group naming it — so Googlebot, bingbot, Yeti and all six AI crawlers inherited nothing from the `*` group and were being handed /api/, /admin/ and /_next/. That is what alpha.moss.land serves today: User-Agent: Googlebot Allow: / Named groups now repeat the same disallow list. Also `host: SITE.baseUrl` emitted "Host: https://alpha.moss.land". The Host directive takes a bare hostname; with a scheme it is invalid and crawlers drop the line. cron timezone The header claimed the box was "the production Mac mini" on KST. It is neither — alpha-web runs on a Linux VM and `timedatectl` reports Etc/UTC. The 2026-05-07 change rewrote every schedule into KST wall-clock against that assumption, moving the jobs 9 hours the wrong way: the "08:30 KST" daily brief fires at 17:30 KST. All 12 schedules converted to UTC with the intended KST time annotated inline, and the header now states the host timezone as verified. Compiles and typechecks; alpha's full build needs a better-sqlite3 native binding this machine lacks. The robots output shape is the same one running in production on media.moss.land. Co-Authored-By: Claude Opus 5 --- app/robots.ts | 68 ++++++++++++++++++++++++-------------------- ecosystem.config.cjs | 42 ++++++++++++++++----------- 2 files changed, 63 insertions(+), 47 deletions(-) diff --git a/app/robots.ts b/app/robots.ts index 81dcd81..e8f82d3 100644 --- a/app/robots.ts +++ b/app/robots.ts @@ -7,41 +7,47 @@ import { SITE } from "@/lib/seo"; * 검색 노출용 봇 = 항상 허용 (visibility 핵심) * 사용자 요청 브라우징 봇 = 항상 허용 * 학습 데이터 수집 봇 = 옵트인 (장기 자산) + * + * 이름을 명시한 봇들도 `*` 그룹과 **똑같은 disallow** 를 받아야 한다. + * robots.txt 의 그룹 매칭은 배타적이라, 크롤러는 자기를 지목한 가장 + * 구체적인 그룹 하나만 읽고 나머지는 통째로 무시한다. 예전처럼 + * `{ userAgent: "Googlebot", allow: "/" }` 만 두면 Googlebot 은 `*` 의 + * Disallow 를 상속하지 않고 /api/·/admin/·/_next/ 를 전부 크롤한다. + * 실제로 그렇게 서빙되고 있었다. */ -export default function robots(): MetadataRoute.Robots { - return { - rules: [ - // 일반 봇 + 검색엔진 (Googlebot/bingbot/Yeti 등 포함) - { - userAgent: "*", - allow: "/", - disallow: ["/api/", "/admin/", "/_next/"], - }, - - // === 검색 노출용 봇 (must-allow) === - { userAgent: "OAI-SearchBot", allow: "/" }, - { userAgent: "Claude-SearchBot", allow: "/" }, - { userAgent: "PerplexityBot", allow: "/" }, - // === 사용자 요청 브라우징 봇 === - { userAgent: "ChatGPT-User", allow: "/" }, - { userAgent: "Claude-User", allow: "/" }, +// 정책상 전부 허용이지만, 허용을 "명시"하는 데 의미가 있는 봇들. +// (특히 학습 크롤러는 기본 opt-out 으로 해석될 여지가 있어 명시한다.) +const NAMED_BOTS = [ + // 검색 노출용 봇 (must-allow) + "OAI-SearchBot", + "Claude-SearchBot", + "PerplexityBot", + // 사용자 요청 브라우징 봇 + "ChatGPT-User", + "Claude-User", + // 학습 데이터 수집 봇 (opt-in 우위) + "GPTBot", + "ClaudeBot", + "Google-Extended", + "CCBot", + // 검색엔진 + "Googlebot", + "bingbot", + "Yeti", +]; - // === 학습 데이터 수집 봇 (opt-in 우위) === - { userAgent: "GPTBot", allow: "/" }, - { userAgent: "ClaudeBot", allow: "/" }, - { userAgent: "Google-Extended", allow: "/" }, - { userAgent: "CCBot", allow: "/" }, +const DISALLOW = ["/api/", "/admin/", "/_next/"]; - // === 검색엔진 === - { userAgent: "Googlebot", allow: "/" }, - { userAgent: "bingbot", allow: "/" }, - { userAgent: "Yeti", allow: "/" }, - ], - sitemap: [ - `${SITE.baseUrl}/sitemap.xml`, - `${SITE.baseUrl}/rss.xml`, +export default function robots(): MetadataRoute.Robots { + return { + rules: [ + { userAgent: "*", allow: "/", disallow: DISALLOW }, + { userAgent: NAMED_BOTS, allow: "/", disallow: DISALLOW }, ], - host: SITE.baseUrl, + sitemap: [`${SITE.baseUrl}/sitemap.xml`, `${SITE.baseUrl}/rss.xml`], + // Host 디렉티브는 스킴 없는 순수 호스트명이다. `https://alpha.moss.land` + // 로 나가고 있었는데 그건 문법상 무효라 크롤러가 그냥 버린다. + host: SITE.baseUrl.replace(/^https?:\/\//, ""), }; } diff --git a/ecosystem.config.cjs b/ecosystem.config.cjs index 869e320..63f0c0a 100644 --- a/ecosystem.config.cjs +++ b/ecosystem.config.cjs @@ -4,10 +4,20 @@ * `cwd: __dirname` resolves to wherever this file lives, so the same * config works on any host (Mac mini, Lightsail, VPS, etc.). * - * Cron schedules are interpreted in the host's *local* timezone — on the - * production Mac mini that's KST. Times below are written in KST and the - * comments explicitly state so. (An earlier version assumed the schedules - * were UTC and silently fired 9 hours early. Fixed 2026-05-07.) + * Cron schedules are interpreted in the host's *local* timezone, and the + * production host — a Linux VM, not a Mac mini — is on **Etc/UTC**. + * Verified with `timedatectl`, not assumed. + * + * So every `cron_restart` below is written in **UTC**, and the comment on + * each app states the KST time it is meant to land on. KST is UTC+9, so a + * morning-KST job runs the previous evening in UTC (06:00 KST = 21:00 UTC + * the day before). + * + * History: the 2026-05-07 note here claimed the box was a KST Mac mini and + * rewrote every schedule into KST wall-clock. The box was already UTC, so + * that change moved the jobs 9 hours the wrong way — the daily brief + * "08:30 KST" was firing at 17:30 KST. Corrected, this time against the + * host's actual timezone. */ const ROOT = __dirname; @@ -35,7 +45,7 @@ module.exports = { cwd: ROOT, script: "./node_modules/.bin/tsx", args: "scripts/indexnow-cron.ts", - cron_restart: "0 4 * * 1", + cron_restart: "0 19 * * 0", // UTC = 04:00 KST 익일 autorestart: false, env: { NODE_ENV: "production" }, }, @@ -45,7 +55,7 @@ module.exports = { cwd: ROOT, script: "./node_modules/.bin/tsx", args: "scripts/fetch-macro.ts", - cron_restart: "0 6 * * *", + cron_restart: "0 21 * * *", // UTC = 06:00 KST 익일 autorestart: false, env: { NODE_ENV: "production" }, }, @@ -55,7 +65,7 @@ module.exports = { cwd: ROOT, script: "./node_modules/.bin/tsx", args: "scripts/generate-synthesis.ts top --limit=30", - cron_restart: "0 7 * * *", + cron_restart: "0 22 * * *", // UTC = 07:00 KST 익일 autorestart: false, env: { NODE_ENV: "production" }, }, @@ -67,7 +77,7 @@ module.exports = { cwd: ROOT, script: "./node_modules/.bin/tsx", args: "scripts/seed-qa-dynamic.ts --limit=20", - cron_restart: "15 7 * * *", + cron_restart: "15 22 * * *", // UTC = 07:15 KST 익일 autorestart: false, env: { NODE_ENV: "production" }, }, @@ -78,7 +88,7 @@ module.exports = { cwd: ROOT, script: "./node_modules/.bin/tsx", args: "scripts/generate-brief.ts", - cron_restart: "30 8 * * *", + cron_restart: "30 23 * * *", // UTC = 08:30 KST 익일 autorestart: false, env: { NODE_ENV: "production" }, }, @@ -89,7 +99,7 @@ module.exports = { cwd: ROOT, script: "./node_modules/.bin/tsx", args: "scripts/translate-briefs.ts --days=14", - cron_restart: "40 8 * * *", + cron_restart: "40 23 * * *", // UTC = 08:40 KST 익일 autorestart: false, env: { NODE_ENV: "production" }, }, @@ -100,7 +110,7 @@ module.exports = { cwd: ROOT, script: "./node_modules/.bin/tsx", args: "scripts/persona-tick.ts --pages=10", - cron_restart: "0 9 * * *", + cron_restart: "0 0 * * *", // UTC = 09:00 KST autorestart: false, env: { NODE_ENV: "production" }, }, @@ -110,7 +120,7 @@ module.exports = { cwd: ROOT, script: "./node_modules/.bin/tsx", args: "scripts/persona-replies.ts --max=8", - cron_restart: "0 12 * * *", + cron_restart: "0 3 * * *", // UTC = 12:00 KST autorestart: false, env: { NODE_ENV: "production" }, }, @@ -122,7 +132,7 @@ module.exports = { cwd: ROOT, script: "./node_modules/.bin/tsx", args: "scripts/track-calls.ts", - cron_restart: "0 13 * * *", + cron_restart: "0 4 * * *", // UTC = 13:00 KST autorestart: false, env: { NODE_ENV: "production" }, }, @@ -132,7 +142,7 @@ module.exports = { cwd: ROOT, script: "./node_modules/.bin/tsx", args: "scripts/generate-why-moved.ts", - cron_restart: "45 8 * * *", + cron_restart: "45 23 * * *", // UTC = 08:45 KST 익일 autorestart: false, env: { NODE_ENV: "production" }, }, @@ -143,7 +153,7 @@ module.exports = { cwd: ROOT, script: "./node_modules/.bin/tsx", args: "scripts/generate-connections.ts top --limit=80", - cron_restart: "15 7 * * *", + cron_restart: "15 22 * * *", // UTC = 07:15 KST 익일 autorestart: false, env: { NODE_ENV: "production" }, }, @@ -155,7 +165,7 @@ module.exports = { cwd: ROOT, script: "./node_modules/.bin/tsx", args: "scripts/audit-auto.ts", - cron_restart: "0 11 * * 1", + cron_restart: "0 2 * * 1", // UTC = 11:00 KST autorestart: false, env: { NODE_ENV: "production" }, },