diff --git a/apps/ratewise/public/rates.json b/apps/ratewise/public/rates.json index 485638f3..9d310eb6 100644 --- a/apps/ratewise/public/rates.json +++ b/apps/ratewise/public/rates.json @@ -1,137 +1,137 @@ { - "timestamp": "2026-04-09T23:51:00.840Z", - "updateTime": "2026/04/10 07:51:00", + "timestamp": "2026-04-10T08:06:27.017Z", + "updateTime": "2026/04/10 16:06:27", "source": "Taiwan Bank (臺灣銀行牌告匯率)", "sourceUrl": "https://rate.bot.com.tw/xrt", "base": "TWD", "rates": { - "USD": 32.02, - "HKD": 4.101, - "GBP": 43.56, - "AUD": 22.87, - "CAD": 23.41, - "SGD": 25.28, - "CHF": 40.54, - "JPY": 0.2033, - "NZD": 18.96, - "THB": 1.0516, - "PHP": 0.5982, - "IDR": 0.00218, - "EUR": 37.68, - "KRW": 0.0237, - "VND": 0.00137, - "MYR": 8.471, - "CNY": 4.717 + "USD": 32.005, + "HKD": 4.099, + "GBP": 43.5, + "AUD": 22.77, + "CAD": 23.34, + "SGD": 25.21, + "CHF": 40.53, + "JPY": 0.2027, + "NZD": 18.89, + "THB": 1.0476, + "PHP": 0.5952, + "IDR": 0.00223, + "EUR": 37.59, + "KRW": 0.02351, + "VND": 0.00139, + "MYR": 8.491, + "CNY": 4.708 }, "details": { "USD": { "name": "美金", "spot": { - "buy": 31.675, - "sell": 31.825 + "buy": 31.685, + "sell": 31.785 }, "cash": { - "buy": 31.35, - "sell": 32.02 + "buy": 31.335, + "sell": 32.005 } }, "HKD": { "name": "港幣", "spot": { - "buy": 4.018, - "sell": 4.088 + "buy": 4.021, + "sell": 4.081 }, "cash": { - "buy": 3.897, - "sell": 4.101 + "buy": 3.895, + "sell": 4.099 } }, "GBP": { "name": "英鎊", "spot": { - "buy": 42.335, - "sell": 42.965 + "buy": 42.39, + "sell": 42.79 }, "cash": { - "buy": 41.44, - "sell": 43.56 + "buy": 41.38, + "sell": 43.5 } }, "AUD": { "name": "澳幣", "spot": { - "buy": 22.305, - "sell": 22.65 + "buy": 22.28, + "sell": 22.48 }, "cash": { - "buy": 22.09, - "sell": 22.87 + "buy": 21.99, + "sell": 22.77 } }, "CAD": { "name": "加拿大幣", "spot": { "buy": 22.83, - "sell": 23.16 + "sell": 23.03 }, "cash": { - "buy": 22.5, - "sell": 23.41 + "buy": 22.43, + "sell": 23.34 } }, "SGD": { "name": "新加坡幣", "spot": { - "buy": 24.84, - "sell": 25.06 + "buy": 24.79, + "sell": 24.97 }, "cash": { - "buy": 24.37, - "sell": 25.28 + "buy": 24.3, + "sell": 25.21 } }, "CHF": { "name": "瑞士法郎", "spot": { - "buy": 39.95, - "sell": 40.34 + "buy": 40.01, + "sell": 40.26 }, "cash": { - "buy": 39.34, - "sell": 40.54 + "buy": 39.33, + "sell": 40.53 } }, "JPY": { "name": "日圓", "spot": { - "buy": 0.1973, - "sell": 0.2023 + "buy": 0.1972, + "sell": 0.2012 }, "cash": { - "buy": 0.1905, - "sell": 0.2033 + "buy": 0.1899, + "sell": 0.2027 } }, "NZD": { "name": "紐元", "spot": { - "buy": 18.44, - "sell": 18.74 + "buy": 18.42, + "sell": 18.62 }, "cash": { - "buy": 18.11, - "sell": 18.96 + "buy": 18.04, + "sell": 18.89 } }, "THB": { "name": "泰銖", "spot": { - "buy": 0.972, - "sell": 1.018 + "buy": 0.971, + "sell": 1.011 }, "cash": { - "buy": 0.8616, - "sell": 1.0516 + "buy": 0.8576, + "sell": 1.0476 } }, "PHP": { @@ -141,8 +141,8 @@ "sell": null }, "cash": { - "buy": 0.4662, - "sell": 0.5982 + "buy": 0.4632, + "sell": 0.5952 } }, "IDR": { @@ -152,19 +152,19 @@ "sell": null }, "cash": { - "buy": 0.00148, - "sell": 0.00218 + "buy": 0.00153, + "sell": 0.00223 } }, "EUR": { "name": "歐元", "spot": { - "buy": 36.855, - "sell": 37.455 + "buy": 36.87, + "sell": 37.27 }, "cash": { - "buy": 36.34, - "sell": 37.68 + "buy": 36.25, + "sell": 37.59 } }, "KRW": { @@ -174,8 +174,8 @@ "sell": null }, "cash": { - "buy": 0.0198, - "sell": 0.0237 + "buy": 0.01961, + "sell": 0.02351 } }, "VND": { @@ -185,8 +185,8 @@ "sell": null }, "cash": { - "buy": 0.00096, - "sell": 0.00137 + "buy": 0.00098, + "sell": 0.00139 } }, "MYR": { @@ -196,19 +196,19 @@ "sell": null }, "cash": { - "buy": 6.946, - "sell": 8.471 + "buy": 6.966, + "sell": 8.491 } }, "CNY": { "name": "人民幣", "spot": { - "buy": 4.622, - "sell": 4.682 + "buy": 4.618, + "sell": 4.668 }, "cash": { - "buy": 4.555, - "sell": 4.717 + "buy": 4.546, + "sell": 4.708 } } } diff --git a/apps/ratewise/public/robots.txt b/apps/ratewise/public/robots.txt index fea68c9b..ec36f38a 100644 --- a/apps/ratewise/public/robots.txt +++ b/apps/ratewise/public/robots.txt @@ -1,7 +1,3 @@ -# RateWise — Robots Exclusion Protocol -# https://app.haotool.org/ratewise/ -# 最後更新：2026-04-09 - User-agent: * Allow: / Disallow: /ratewise/sw.js @@ -10,9 +6,6 @@ Disallow: /ratewise/theme-showcase/ Disallow: /ratewise/color-scheme/ Disallow: /ratewise/update-prompt-test/ Disallow: /ratewise/ui-showcase/ -# 首頁 deep-link 僅供分享，不作為索引目標。 -# 幣對金額頁是否索引由頁面層 canonical 與 robots 決定。 -# robots.txt 僅控制 crawl 範圍。 Disallow: /ratewise/? Sitemap: https://app.haotool.org/ratewise/sitemap.xml @@ -47,6 +40,9 @@ Allow: / User-agent: Google-Extended Allow: / +User-agent: Google-CloudVertexBot +Allow: / + User-agent: GrokBot Allow: / @@ -56,12 +52,18 @@ Allow: / User-agent: YouBot Allow: / +User-agent: PhindBot +Allow: / + User-agent: DuckAssistBot Allow: / User-agent: Amazonbot Allow: / +User-agent: Applebot +Allow: / + User-agent: Applebot-Extended Allow: / @@ -71,9 +73,24 @@ Allow: / User-agent: Bytespider Allow: / +User-agent: PetalBot +Allow: / + +User-agent: MistralAI-User +Allow: / + +User-agent: Manus-User +Allow: / + User-agent: Meta-ExternalAgent Allow: / +User-agent: Meta-ExternalFetcher +Allow: / + +User-agent: FacebookBot +Allow: / + User-agent: facebookexternalhit Allow: / @@ -82,3 +99,24 @@ Allow: / User-agent: LinkedInBot Allow: / + +User-agent: Cloudflare-AutoRAG +Allow: / + +User-agent: Anchor Browser +Allow: / + +User-agent: archive.org_bot +Allow: / + +User-agent: Terracotta Bot +Allow: / + +User-agent: Timpibot +Allow: / + +User-agent: ProRataInc +Allow: / + +User-agent: Novellum AI Crawl +Allow: / diff --git a/apps/ratewise/scripts/generate-robots-txt.mjs b/apps/ratewise/scripts/generate-robots-txt.mjs index f1ba0382..5a1429b2 100644 --- a/apps/ratewise/scripts/generate-robots-txt.mjs +++ b/apps/ratewise/scripts/generate-robots-txt.mjs @@ -8,7 +8,11 @@ const ROOT = resolve(__dirname, '..'); const BUILD_DATE = new Date().toISOString().split('T')[0]; const SITEMAP_URL = `${SITE_CONFIG.url}sitemap.xml`; -const AI_BOTS = [ +// AI-specific bots that need explicit Allow rules +// Note: Googlebot and Bingbot are NOT included here because they should follow +// the User-agent: * rules (including Disallow directives). Adding dedicated +// User-agent blocks for them would override the wildcard disallow rules. +const AI_SEARCH_BOTS = [ 'GPTBot', 'OAI-SearchBot', 'ChatGPT-User', @@ -19,18 +23,35 @@ const AI_BOTS = [ 'PerplexityBot', 'Perplexity-User', 'Google-Extended', + 'Google-CloudVertexBot', 'GrokBot', 'cohere-ai', 'YouBot', + 'PhindBot', 'DuckAssistBot', 'Amazonbot', + 'Applebot', 'Applebot-Extended', 'CCBot', 'Bytespider', + 'PetalBot', + 'MistralAI-User', + 'Manus-User', + 'Meta-ExternalAgent', + 'Meta-ExternalFetcher', + 'FacebookBot', + 'facebookexternalhit', + 'Twitterbot', + 'LinkedInBot', + 'Cloudflare-AutoRAG', + 'Anchor Browser', + 'archive.org_bot', + 'Terracotta Bot', + 'Timpibot', + 'ProRataInc', + 'Novellum AI Crawl', ]; -const SOCIAL_BOTS = ['Meta-ExternalAgent', 'facebookexternalhit', 'Twitterbot', 'LinkedInBot']; - function sectionForBots(bots) { return bots.map((bot) => `User-agent: ${bot}\nAllow: /`).join('\n\n'); } @@ -41,26 +62,17 @@ function buildDisallowRules(paths) { .join('\n'); } -const robotsTxt = `# RateWise — Robots Exclusion Protocol -# ${SITE_CONFIG.url} -# 最後更新：${BUILD_DATE} - -User-agent: * +const robotsTxt = `User-agent: * Allow: / Disallow: /ratewise/sw.js Disallow: /ratewise/workbox-*.js ${buildDisallowRules(DEV_ONLY_PATHS)} -# 首頁 deep-link 僅供分享，不作為索引目標。 -# 幣對金額頁是否索引由頁面層 canonical 與 robots 決定。 -# robots.txt 僅控制 crawl 範圍。 Disallow: /ratewise/? Sitemap: ${SITEMAP_URL} -${sectionForBots(AI_BOTS)} - -${sectionForBots(SOCIAL_BOTS)} +${sectionForBots(AI_SEARCH_BOTS)} `; writeFileSync(resolve(ROOT, 'public/robots.txt'), robotsTxt.trimEnd() + '\n'); -console.log('✅ robots.txt 已由 SSOT 重新生成'); +console.log('✅ robots.txt generated'); diff --git a/docs/SEO_MASTER_SSOT.md b/docs/SEO_MASTER_SSOT.md index 33d94081..5bc5a743 100644 --- a/docs/SEO_MASTER_SSOT.md +++ b/docs/SEO_MASTER_SSOT.md @@ -578,7 +578,7 @@ AI 引擎傾向引用每頁擁有 15+ 連結實體的頁面。幣對頁應自然 ## 8. robots.txt 規範 -### 8.1 現況（已確認正確的規則） +### 8.1 現況（v2.24.0 更新，全面開放 AI 爬蟲） ``` User-agent: * @@ -592,31 +592,67 @@ Disallow: /ratewise/ui-showcase/ Disallow: /ratewise/? ``` -### 8.2 AI 爬蟲規則（必須完整） - -robots.txt 必須明確 Allow 以下 AI 爬蟲（目前已有，需確認完整）： - -| 爬蟲名稱 | 平台 | 用途 | -| ----------------- | ------------ | ------------------------------- | -| `GPTBot` | OpenAI | ChatGPT 訓練 | -| `OAI-SearchBot` | OpenAI | ChatGPT 搜尋引用 | -| `ChatGPT-User` | OpenAI | ChatGPT 用戶搜尋 | -| `ClaudeBot` | Anthropic | Claude 訓練爬蟲 | -| `Claude-User` | Anthropic | **Claude 搜尋引用（必須允許）** | -| `anthropic-ai` | Anthropic | Anthropic 通用 | -| `PerplexityBot` | Perplexity | Perplexity 引用 | -| `Google-Extended` | Google | Gemini / AI Overviews | -| `Bingbot` | Microsoft | Copilot（透過 Bing） | -| `Amazonbot` | Amazon | Alexa / AWS AI | -| `Bytespider` | ByteDance | TikTok 系列 AI | -| `CCBot` | Common Crawl | 開放資料（可選允許） | - -> **策略說明**：允許訓練爬蟲（如 ClaudeBot）可增加 AI 訓練資料中的品牌認知，雖然不直接控制引用，但有助於長期品牌識別。 - -### 8.3 需確認的規則 - -- 確認 `Claude-User` 是否已在 `generate-robots-txt.mjs` 的允許清單中 -- 確認 `/ratewise/?` 的 Disallow 不影響幣對頁的 `?amount=` 參數索引（已確認正確） +### 8.2 AI 爬蟲規則（2026-04-10 完整清單，共 37 個） + +robots.txt 明確 Allow 以下 AI 爬蟲，確保最大 AI 搜尋可見度： + +| 爬蟲名稱 | 平台 | 用途 | +| ----------------------- | ---------------- | --------------------- | +| `GPTBot` | OpenAI | ChatGPT 訓練 | +| `OAI-SearchBot` | OpenAI | ChatGPT 搜尋引用 | +| `ChatGPT-User` | OpenAI | ChatGPT 用戶搜尋 | +| `ClaudeBot` | Anthropic | Claude 訓練爬蟲 | +| `Claude-User` | Anthropic | Claude 搜尋引用 | +| `Claude-SearchBot` | Anthropic | Claude 搜尋索引 | +| `anthropic-ai` | Anthropic | Anthropic 通用 | +| `PerplexityBot` | Perplexity | Perplexity 引用 | +| `Perplexity-User` | Perplexity | Perplexity 用戶搜尋 | +| `Google-Extended` | Google | Gemini / AI Overviews | +| `Google-CloudVertexBot` | Google | Vertex AI | +| `GrokBot` | xAI | Grok AI | +| `cohere-ai` | Cohere | Cohere AI 產品 | +| `YouBot` | You.com | You.com AI 搜尋 | +| `PhindBot` | Phind | Phind 開發者 AI 搜尋 | +| `DuckAssistBot` | DuckDuckGo | DuckDuckGo AI 助手 | +| `Amazonbot` | Amazon | Alexa / AWS AI | +| `Applebot` | Apple | Apple 搜尋 + Siri | +| `Applebot-Extended` | Apple | Apple Intelligence | +| `CCBot` | Common Crawl | 開放資料集 | +| `Bytespider` | ByteDance | TikTok 系列 AI | +| `PetalBot` | Huawei | Huawei AI 搜尋 | +| `MistralAI-User` | Mistral | Mistral AI 助手 | +| `Manus-User` | Manus | Manus AI 助手 | +| `Meta-ExternalAgent` | Meta | Meta AI 訓練（Llama） | +| `Meta-ExternalFetcher` | Meta | Meta AI 助手 | +| `FacebookBot` | Meta | Facebook AI | +| `facebookexternalhit` | Meta | Facebook 連結預覽 | +| `Twitterbot` | X (Twitter) | Twitter 連結預覽 | +| `LinkedInBot` | LinkedIn | LinkedIn 連結預覽 | +| `Cloudflare-AutoRAG` | Cloudflare | Cloudflare AI | +| `Anchor Browser` | Anchor | Anchor AI 爬蟲 | +| `archive.org_bot` | Internet Archive | 網路存檔 | +| `Terracotta Bot` | Ceramic | 搜尋引擎爬蟲 | +| `Timpibot` | Timpi | Timpi AI 爬蟲 | +| `ProRataInc` | ProRata.ai | ProRata AI 爬蟲 | +| `Novellum AI Crawl` | Novellum | Novellum AI 爬蟲 | + +### 8.3 不需專門規則的爬蟲 + +以下爬蟲**不需要**專門的 `User-agent` 區塊，因為它們會遵循 `User-agent: *` 的規則： + +| 爬蟲名稱 | 平台 | 原因 | +| ----------- | --------- | ---------------------------------------------------- | +| `Googlebot` | Google | 主要搜尋爬蟲，需遵循 Disallow 規則以避免索引開發頁面 | +| `Bingbot` | Microsoft | 主要搜尋爬蟲，需遵循 Disallow 規則以避免索引開發頁面 | + +> **重要**：為這些爬蟲添加專門的 `User-agent` 區塊會覆蓋 `User-agent: *` 的 Disallow 規則，導致開發頁面被索引。 + +### 8.4 策略說明 + +- **選擇性開放策略**：AI 爬蟲明確允許，傳統搜尋爬蟲遵循通用規則 +- **訓練 vs 搜尋**：訓練爬蟲（GPTBot、ClaudeBot）增加品牌認知；搜尋爬蟲（OAI-SearchBot、Claude-SearchBot）直接影響引用 +- **Disallow 保護**：`Googlebot` 和 `Bingbot` 遵循 `User-agent: *` 的 Disallow 規則，防止開發頁面被索引 +- **SSOT 管理**：由 `generate-robots-txt.mjs` 自動生成，禁止手動修改 ---