diff --git a/backend/internal/api/compress.go b/backend/internal/api/compress.go index ac4be6e..53ce0f1 100644 --- a/backend/internal/api/compress.go +++ b/backend/internal/api/compress.go @@ -246,7 +246,7 @@ func etagMatches(header, etag string) bool { // - /api/health and /api/admin/* are never cached (sensitive / monitoring) // - /api/categories is cached for 5 minutes (rarely changes) // - /api/news/trending is server-cached for minutes, so browsers and CDNs may reuse it -// - /robots.txt is cached for 1 day +// - /robots.txt is marked no-cache so crawlers always receive real-time rules // - /sitemap.xml is cached for 1 hour // - static assets (including /site.webmanifest, favicons, /uploads/) are cached for 30 days // - immutable assets (/_app/immutable/*) are cached for 1 year @@ -263,7 +263,7 @@ func (s *Server) cacheControl(next http.Handler) http.Handler { case path == "/api/news/trending": w.Header().Set("Cache-Control", "public, max-age=60, stale-while-revalidate=60") case path == "/robots.txt": - w.Header().Set("Cache-Control", "public, max-age=86400") + w.Header().Set("Cache-Control", "no-cache") case path == "/sitemap.xml": w.Header().Set("Cache-Control", "public, max-age=3600") case strings.HasPrefix(path, "/_app/immutable/"): diff --git a/backend/internal/api/handlers.go b/backend/internal/api/handlers.go index 8300b2d..e31ff63 100644 --- a/backend/internal/api/handlers.go +++ b/backend/internal/api/handlers.go @@ -369,7 +369,7 @@ func (s *Server) handleRobotsTXT(w http.ResponseWriter, r *http.Request) { "User-agent: meta-externalagent\nDisallow: /\n\n" + "Sitemap: " + origin + "/sitemap.xml\n" w.Header().Set("Content-Type", "text/plain; charset=utf-8") - w.Header().Set("Cache-Control", "public, max-age=86400") + w.Header().Set("Cache-Control", "no-cache") _, _ = w.Write([]byte(body)) } diff --git a/backend/internal/api/handlers_test.go b/backend/internal/api/handlers_test.go index cbf0319..4ed58d1 100644 --- a/backend/internal/api/handlers_test.go +++ b/backend/internal/api/handlers_test.go @@ -1792,7 +1792,7 @@ func TestCacheControlHeaders(t *testing.T) { {"/api/healthz", "no-store"}, {"/api/news/trending", "public, max-age=60, stale-while-revalidate=60"}, {"/api/categories", "public, max-age=300, stale-while-revalidate=60"}, - {"/robots.txt", "public, max-age=86400"}, + {"/robots.txt", "no-cache"}, {"/sitemap.xml", "public, max-age=3600"}, }