diff --git a/internal/api/api.go b/internal/api/api.go index b2264b8..97fa2ad 100644 --- a/internal/api/api.go +++ b/internal/api/api.go @@ -67,7 +67,7 @@ func (a *API) Register(app core.App) { }) e.Router.GET("/robots.txt", func(event *core.RequestEvent) error { event.Response.Header().Set("Cache-Control", "no-store") - return event.String(http.StatusOK, "User-agent: *\nDisallow:\n") + return event.String(http.StatusOK, "User-agent: *\nContent-Signal: search=no, ai-input=no, ai-train=no, use=immediate\nDisallow:\n") }) e.Router.POST("/api/payments", a.createPayment).Bind(apis.BodyLimit(maxPaymentRequestBytes)) e.Router.GET("/api/payments/{id}", a.getPayment) diff --git a/internal/api/operations_test.go b/internal/api/operations_test.go index e51aa1e..663dfe4 100644 --- a/internal/api/operations_test.go +++ b/internal/api/operations_test.go @@ -280,7 +280,7 @@ func TestOperatorSPAUsesSecurityHeadersAndRejectsUnknownBrowserRoutes(t *testing } } res, robots := fixture.request(t, http.MethodGet, "/robots.txt", nil, "", false) - if res.StatusCode != http.StatusOK || !bytes.Contains(robots, []byte("User-agent: *")) || res.Header.Get("X-Robots-Tag") == "" { + if res.StatusCode != http.StatusOK || !bytes.Contains(robots, []byte("User-agent: *")) || !bytes.Contains(robots, []byte("Content-Signal: search=no")) || res.Header.Get("X-Robots-Tag") == "" { t.Fatalf("robots status=%d header=%q body=%s", res.StatusCode, res.Header.Get("X-Robots-Tag"), robots) } for _, path := range []string{"/contact/", "/sitemap.xml"} {