From c2b4b91ef151e8e9b33a7dfe4c5298fdbe88a889 Mon Sep 17 00:00:00 2001 From: Teodor Calin Date: Thu, 17 Sep 2026 21:29:53 +0300 Subject: [PATCH 01/22] Document public agent setup and add Cosift usage skill --- docs/AGENT-SETUP.md | 129 +++++++++++++++++++++++++++++++++++++++++ skills/cosift/SKILL.md | 91 +++++++++++++++++++++++++++++ 2 files changed, 220 insertions(+) create mode 100644 docs/AGENT-SETUP.md create mode 100644 skills/cosift/SKILL.md diff --git a/docs/AGENT-SETUP.md b/docs/AGENT-SETUP.md new file mode 100644 index 0000000..1c49924 --- /dev/null +++ b/docs/AGENT-SETUP.md @@ -0,0 +1,129 @@ +# Use Cosift from your agent + +[Open Cosift](https://cosift.pilotprotocol.network/) to sign in with an email code, +search, get a cited answer, run research, save requests, or contribute public pages. +The corpus is growing and incomplete. Try any topic; a coverage miss is useful +feedback, and good public sources from any field are welcome. + +## Install and connect + +Run the public installer in your own terminal: + +```sh +curl -fsSL https://raw.githubusercontent.com/pilot-protocol/cosift-install/v1/install.sh | sh +``` + +It detects supported agents, confirms your email, configures their MCP connection, +and installs the onboarding skill. Onboarding can suggest interests from local +agent history; review what you share. Never submit raw private history as a +contribution. See the [installer documentation](https://github.com/pilot-protocol/cosift-install) +for supported harnesses and options. + +For a manual connection, use streamable HTTP: + +| Setting | Value | +| --- | --- | +| MCP URL | `https://cosift-mcp-udik5erlkq-uw.a.run.app/v1/mcp` | +| Authentication | `Authorization: Bearer ` | + +Use the token obtained by the installer in your agent's private configuration. +Do not paste it into chats, commit it, or send it to another origin. Public +transport still requires a Cosift account token for every MCP tool. + +## Tools your agent can use + +The examples below are tool arguments, not shell commands. + +| Tool | Example arguments | Result | +| --- | --- | --- | +| `cosift_search` | `{"query":"Go modules tutorial","k":3}` | Source URLs, titles and excerpts; fewer hits or no hits are possible. | +| `cosift_lookup` | `{"topic":"Go dependency management"}` | Checks curated article coverage; a miss can include retry guidance. | +| `cosift_request` | `{"topic":"Go dependency management","why":"Compare module versioning approaches"}` | Records an explicit request for coverage. | +| `cosift_topics` | `{"action":"list"}` | Lists this account's followed/requested topics. | +| `cosift_topics` | `{"action":"add","topics":["Go dependency management"]}` | Follows interests; use `remove` with the same shape to unfollow. | + +Search returns sources, not a synthesized answer. Use the web app or CLI for +Answer and Research. Curated article generation is not enabled in the current +release: lookup/request connects the coverage and demand workflow without +promising an article or delivery date. Following a topic does not request an +article, and unfollowing does not erase previously requested-topic history. + +## Add the general Cosift skill + +The installer already supplies onboarding. The separate +[Cosift usage skill](../skills/cosift/SKILL.md) teaches agents how to search, +handle coverage misses, follow topics, and contribute with the CLI. +[Download its SKILL.md](https://raw.githubusercontent.com/pilot-protocol/cosift/main/skills/cosift/SKILL.md). +Place that file at `cosift/SKILL.md` inside your agent's configured skill directory, +then reload the agent. This does not require uploading a token to the skill file. + +## Authenticated CLI + +Use [Cosift v0.2.7 or a newer stable release](https://github.com/pilot-protocol/cosift/releases/latest) +for your OS and architecture. Put the binary on your PATH. After installer login, +the CLI discovers the private `cosift/community-session.json` under +`$XDG_CONFIG_HOME`, or `~/.config` when unset. An explicit `-session-file` can select +another saved session. Keep session files private (mode `0600`). + +```sh +cosift request -server https://cosift.pilotprotocol.network -query "Go modules tutorial" +cosift request -server https://cosift.pilotprotocol.network -mode answer -query "What is a Go module?" +cosift request -server https://cosift.pilotprotocol.network -mode research -query "How does Go module versioning work?" +cosift contribute -server https://cosift.pilotprotocol.network https://go.dev/doc/modules/ +cosift contribute -server https://cosift.pilotprotocol.network -csv sources.csv +cosift contribute -server https://cosift.pilotprotocol.network -credits +``` + +Contributions require login. A CSV can contain one URL per row or a `url`, `urls`, +`webpage`, or `website` column. A request accepts at most 100 URLs and 1 MB. +Put flags before positional URLs. For example: + +```csv +url +https://go.dev/doc/modules/ +https://www.rust-lang.org/learn +``` + +Submit useful public pages you are authorized to share. URL and content checks +screen adult material, harmful content, malware/phishing, spam and low-quality +filler; useful educational, medical and defensive-security material is allowed. +Accepted submissions enter validation; acceptance alone does not mean indexing +or a credit award. Check contribution status in the web app. Checks are automated +and do not guarantee that every unsafe or poor-quality page is detected. + +## Index and embed locally + +The CLI can fetch, extract, chunk and embed a page locally, retain its local index, +then submit its text, metadata and vectors. Configure a local embedding service +matching the destination model and dimensions. Production currently uses +`nomic-embed-text` with 768 dimensions; confirm compatibility before a large run. + +Example `local.json` for an OpenAI-compatible local embedding endpoint: + +```json +{"data_dir":"./cosift-local-data","embeddings":{"url":"http://127.0.0.1:11434/v1","model":"nomic-embed-text","dim":768}} +``` + +```sh +cosift -config local.json contribute -server https://cosift.pilotprotocol.network -index-locally https://go.dev/doc/modules/ +``` + +The server independently checks the source, content safety and quality, model, +dimensions and every vector before reuse. Current limits are 32,000 text bytes, +64 chunks per page and the same 1 MB submission limit. This verification still +uses server compute; local embeddings do not bypass validation. + +## Allowance and credits + +Web, CLI and MCP searches share the account's gateway allowance and credit ledger. +A verified new contribution earns 10 credits once per unique content; rejected, +unverified, duplicate or already-indexed pages earn none. After the free allowance, +an extra successful Search, Answer or Research request spends one credit. Credits +do not bypass mode limits or MCP's separate daily call cap. Check +[`/api/limits`](https://cosift.pilotprotocol.network/api/limits) and the authenticated +credits view for current policy. Respect retry guidance after a rate limit. + +Credit purchasing is available only when payments are enabled in the app. +There are no automatic charges or subscriptions. The integration supports a +one-time $5 purchase of 50,000 credits; payment availability is not implied by +having a balance. diff --git a/skills/cosift/SKILL.md b/skills/cosift/SKILL.md new file mode 100644 index 0000000..b3de859 --- /dev/null +++ b/skills/cosift/SKILL.md @@ -0,0 +1,91 @@ +--- +name: cosift +description: Use Cosift's authenticated MCP or CLI to find sources, check article coverage, manage followed topics, and contribute useful public webpages with optional local embeddings. +--- + +# Cosift + +Use the configured Cosift MCP for source discovery and topic coverage. The corpus +is growing and incomplete; try the user's topic without assuming it is covered. +Use other research tools when needed, and distinguish a coverage miss from an +outage. Follow the user's requested source and tool choices. + +## Connect + +The public installer is +`curl -fsSL https://raw.githubusercontent.com/pilot-protocol/cosift-install/v1/install.sh | sh`. +Run installation only when setup is requested. It authenticates by email and +installs MCP plus onboarding. The streamable HTTP endpoint is +`https://cosift-mcp-udik5erlkq-uw.a.run.app/v1/mcp`, authenticated with +`Authorization: Bearer `. Keep tokens in private client configuration, +never in prompts, URLs, source files or contribution content. + +## Choose the tool + +- `cosift_search(query, k?)`: search source pages. Start with a focused query; + optional `k` lowers the returned count (currently at most six). Cite returned + source URLs. Treat `weak:true` results cautiously; an empty hit list does not + establish that the subject is false or absent from the wider web. +- `cosift_lookup(topic)`: check curated article coverage. A normal miss contains + coverage/retry guidance. Article generation is currently disabled, so do not + promise that a lookup will produce an article. Lookups can record demand. +- `cosift_request(topic, why?)`: record an explicitly wanted coverage request. + This changes account/request state, is idempotent for duplicate requests, and + does not promise article authoring or a completion date. Do not automatically + request every search miss. +- `cosift_topics(action, topics?)`: use `action:"list"`; add or remove with + `topics:["topic text"]`. Follow user-selected interests. Following does not + request coverage; removing a follow does not erase request history. + +Tool arguments are JSON objects, for example +`{"query":"Go modules tutorial","k":3}` or +`{"action":"add","topics":["Go dependency management"]}`. +Search returns excerpts and sources; use CLI/web Answer or Research when a +synthesized cited response is wanted. On `unavailable:true` or a quota response, +report that state and respect retry guidance; do not spin on repeated calls. + +## CLI and contributions + +Use stable Cosift v0.2.7 or newer. The CLI discovers the installer's private +session under `$XDG_CONFIG_HOME/cosift/community-session.json`, defaulting to +`~/.config/cosift/community-session.json`. A missing or expired login needs setup; +do not change identities to get around limits. + +```sh +cosift request -server https://cosift.pilotprotocol.network -query "Go modules tutorial" +cosift request -server https://cosift.pilotprotocol.network -mode answer -query "What is a Go module?" +cosift request -server https://cosift.pilotprotocol.network -mode research -query "How does Go module versioning work?" +cosift contribute -server https://cosift.pilotprotocol.network https://go.dev/doc/modules/ +cosift contribute -server https://cosift.pilotprotocol.network -csv sources.csv +cosift contribute -server https://cosift.pilotprotocol.network -credits +``` + +Contribute when the user wants to share sources. Login is required. Choose useful, +publicly accessible pages on any topic; do not upload private agent history, +credentials, account-only pages or private documents. CSV accepts one URL column +or a recognized `url`/`urls`/`webpage`/`website` header, up to 100 URLs and 1 MB. +Flags precede positional URLs. Submission acceptance means queued for checking; +only verified indexing can earn credits. Check status in the web app. + +For an explicitly requested local-indexing workflow, configure a compatible local +embedding service and use: + +```sh +cosift -config local.json contribute -server https://cosift.pilotprotocol.network -index-locally https://go.dev/doc/modules/ +``` + +This fetches public content and creates a local index, then uploads text, metadata +and vectors. Match the destination model/dimensions; production currently uses +`nomic-embed-text`, 768 dimensions. The server checks source content, safety, +quality and all vectors. Do not modify content or vectors to evade a rejection. + +Verified new content earns 10 credits once; existing, duplicate, rejected and +unverified pages earn none. Extra successful requests spend credits after the +shared free allowance. Web, CLI and MCP searches use the same gateway account +limits; mode caps and MCP's separate daily cap still apply. Inspect the current +credits policy rather than assuming a balance buys unrestricted usage. + +Read the [setup guide](https://github.com/pilot-protocol/cosift/blob/main/docs/AGENT-SETUP.md) +for installation, CSV examples, local embedding configuration and payment +availability. The general usage skill is separate from installed onboarding; +do not start onboarding or inspect session history merely because this skill loads. From 0a6f7990a35e7c0db73e2c56797cfb2a9ad6a6ac Mon Sep 17 00:00:00 2001 From: Teodor Calin Date: Thu, 17 Sep 2026 21:31:38 +0300 Subject: [PATCH 02/22] Require authenticated contributions and expose monthly credit activity --- cmd/cosift/community.go | 9 +- cmd/cosift/community_test.go | 8 +- internal/community/credits.go | 12 ++- internal/community/guest_test.go | 15 +-- internal/community/moderation_test.go | 8 +- internal/community/release_hotfix_test.go | 125 ++++++++++++++++++++++ internal/community/server.go | 41 +++---- 7 files changed, 180 insertions(+), 38 deletions(-) create mode 100644 internal/community/release_hotfix_test.go diff --git a/cmd/cosift/community.go b/cmd/cosift/community.go index 3452b5e..5b83c4f 100644 --- a/cmd/cosift/community.go +++ b/cmd/cosift/community.go @@ -56,7 +56,7 @@ func runCommunity(ctx context.Context, args []string) error { defer client.Close() provider = client } - s, err := community.Open(community.Config{Shared: provider, DataDir: *dir, Backend: *backend, PublicURL: *publicURL, AdminToken: os.Getenv("COSIFT_COMMUNITY_ADMIN_TOKEN"), TrustedProxies: trusted, GuestInterval: *guestInterval, MemberFreeRPM: *freeRPM, SearchRPM: *searchRPM, AnswerRPM: *answerRPM, ResearchPer10Min: *researchLimit, StripeSecretKey: os.Getenv("STRIPE_SECRET_KEY"), StripeWebhookSecret: os.Getenv("STRIPE_WEBHOOK_SECRET")}) + s, err := community.Open(community.Config{GAMeasurementID: os.Getenv("COSIFT_GA_MEASUREMENT_ID"), Shared: provider, DataDir: *dir, Backend: *backend, PublicURL: *publicURL, AdminToken: os.Getenv("COSIFT_COMMUNITY_ADMIN_TOKEN"), TrustedProxies: trusted, GuestInterval: *guestInterval, MemberFreeRPM: *freeRPM, SearchRPM: *searchRPM, AnswerRPM: *answerRPM, ResearchPer10Min: *researchLimit, StripeSecretKey: os.Getenv("STRIPE_SECRET_KEY"), StripeWebhookSecret: os.Getenv("STRIPE_WEBHOOK_SECRET")}) if err != nil { return err } @@ -102,7 +102,7 @@ func runContributeConfigured(ctx context.Context, cfg *config.Config, args []str sessionFile := fs.String("session-file", os.Getenv("COSIFT_SESSION_FILE"), "private saved CLI session (or COSIFT_SESSION_FILE)") login := fs.Bool("login", false, "save an authenticated CLI session") logout := fs.Bool("logout", false, "revoke and delete the saved CLI session") - guest := fs.Bool("guest", false, "submit without login (server guest limits apply)") + guest := fs.Bool("guest", false, "perform Search, Answer or Research without login; contributions require login") if err := fs.Parse(args); err != nil { return err } @@ -146,7 +146,7 @@ func runContributeConfigured(ctx context.Context, cfg *config.Config, args []str return fmt.Errorf("choose COSIFT_TOKEN or a saved session, not both") } if token == "" && !*guest && (*login || *sessionFile == "") && ((*email == "") != (password == "")) { - return fmt.Errorf("set both COSIFT_EMAIL and COSIFT_PASSWORD, or use -guest") + return fmt.Errorf("set both COSIFT_EMAIL and COSIFT_PASSWORD, or use -guest for retrieval") } origin := strings.TrimRight(*server, "/") values := fs.Args() @@ -161,6 +161,9 @@ func runContributeConfigured(ctx context.Context, cfg *config.Config, args []str return fmt.Errorf("login requires COSIFT_TOKEN from cosift-install, or COSIFT_EMAIL and COSIFT_PASSWORD for standalone servers") } // Validate intent before login, reading stdin, or touching the local index. + if *guest && !*requestMode { + return fmt.Errorf("contributions and account operations require login; -guest is only for -request") + } if *requestMode { if *local || *credits || len(values) > 0 || *file != "" { return fmt.Errorf("request cannot be combined with contributions or credits") diff --git a/cmd/cosift/community_test.go b/cmd/cosift/community_test.go index 00b5bec..563b8e2 100644 --- a/cmd/cosift/community_test.go +++ b/cmd/cosift/community_test.go @@ -109,7 +109,7 @@ func TestCommunityContributeCLI(t *testing.T) { } } -func TestCommunityGuestCLI(t *testing.T) { +func TestCommunityGuestContributionCLIRejected(t *testing.T) { t.Setenv("COSIFT_EMAIL", "") t.Setenv("COSIFT_PASSWORD", "") called := 0 @@ -122,10 +122,10 @@ func TestCommunityGuestCLI(t *testing.T) { w.Write([]byte(`{"accepted":1}`)) })) defer backend.Close() - if err := runContribute(context.Background(), []string{"-server", backend.URL, "-guest", "https://example.com/guide"}); err != nil { - t.Fatal(err) + if err := runContribute(context.Background(), []string{"-server", backend.URL, "-guest", "https://example.com/guide"}); err == nil { + t.Fatal("guest contribution was accepted") } - if called != 1 { + if called != 0 { t.Fatalf("requests %d", called) } } diff --git a/internal/community/credits.go b/internal/community/credits.go index a886300..4adebf8 100644 --- a/internal/community/credits.go +++ b/internal/community/credits.go @@ -10,11 +10,19 @@ const contributionReward = 10 func (s *Server) credits(w http.ResponseWriter, r *http.Request, u User) { var balance int - if err := s.db.QueryRowContext(r.Context(), `SELECT COALESCE(sum(delta),0) FROM credit_ledger WHERE user_id=?`, u.ID).Scan(&balance); err != nil { + var earned, purchased, spent int + now := time.Now().UTC() + start := time.Date(now.Year(), now.Month(), 1, 0, 0, 0, 0, time.UTC) + end := start.AddDate(0, 1, 0) + if err := s.db.QueryRowContext(r.Context(), `SELECT COALESCE(sum(delta),0), +COALESCE(sum(CASE WHEN created_at>=? AND created_at=? AND created_at=? AND created_at?`, tokenHash(cookie.Value), time.Now().Unix())) if errors.Is(err, sql.ErrNoRows) { + http.SetCookie(w, &http.Cookie{Name: cookieName, Path: "/", MaxAge: -1, HttpOnly: true, Secure: strings.HasPrefix(s.cfg.PublicURL, "https:"), SameSite: http.SameSiteLaxMode}) problem(w, 401, "session expired; sign in again") return } @@ -619,6 +633,10 @@ func (s *Server) submissions(w http.ResponseWriter, r *http.Request, u User) { respond(w, 200, out) } func (s *Server) submit(w http.ResponseWriter, r *http.Request, u User) { + if u.ID == "" { + problem(w, http.StatusUnauthorized, "contributions require login") + return + } if !s.allow("submit:"+u.ID+":"+s.clientIP(r), 20, time.Minute) { problem(w, 429, "too many submissions; try again in a minute") return @@ -648,10 +666,6 @@ func (s *Server) submit(w http.ResponseWriter, r *http.Request, u User) { err = decode(r, &in) values = in.URLs if len(in.Artifacts) > 0 { - if u.ID == "" { - problem(w, 401, "local indexing contributions require login") - return - } if len(values) > 0 { problem(w, 400, "use URLs or local artifacts") return @@ -681,14 +695,6 @@ func (s *Server) submit(w http.ResponseWriter, r *http.Request, u User) { problem(w, 400, err.Error()) return } - completed := false - if u.ID == "" { - finish, ok := s.reserveGuest(w, r) - if !ok { - return - } - defer func() { finish(completed) }() - } tx, err := s.db.BeginTx(r.Context(), nil) if err != nil { problem(w, 500, "could not save contribution") @@ -703,13 +709,9 @@ func (s *Server) submit(w http.ResponseWriter, r *http.Request, u User) { } accepted := 0 duplicates := 0 - var owner any - if u.ID != "" { - owner = u.ID - } for _, v := range values { id := randomID() - res, e := tx.ExecContext(r.Context(), `INSERT INTO submissions(id,user_id,url,created_at) VALUES(?,?,?,?) ON CONFLICT(user_id,url) DO NOTHING`, id, owner, v, now) + res, e := tx.ExecContext(r.Context(), `INSERT INTO submissions(id,user_id,url,created_at) VALUES(?,?,?,?) ON CONFLICT(user_id,url) DO NOTHING`, id, u.ID, v, now) if e != nil { problem(w, 500, "could not save contribution") return @@ -736,7 +738,6 @@ func (s *Server) submit(w http.ResponseWriter, r *http.Request, u User) { problem(w, 500, "could not save contribution") return } - completed = true respond(w, 202, map[string]any{"accepted": accepted, "duplicates": duplicates, "status": "pending"}) } From 7a5441604b3d26ed064d30c10502fbc5f08eab73 Mon Sep 17 00:00:00 2001 From: Teodor Calin Date: Thu, 17 Sep 2026 21:34:56 +0300 Subject: [PATCH 03/22] Bind contribution delivery to freshly moderated page content --- internal/community/moderation.go | 52 ++++---- internal/community/moderation_binding_test.go | 116 ++++++++++++++++++ internal/community/server.go | 9 +- 3 files changed, 147 insertions(+), 30 deletions(-) create mode 100644 internal/community/moderation_binding_test.go diff --git a/internal/community/moderation.go b/internal/community/moderation.go index f5b3363..6d9ba3b 100644 --- a/internal/community/moderation.go +++ b/internal/community/moderation.go @@ -61,56 +61,56 @@ func newModerationClient() *http.Client { // prevalidate runs before delivery. Only an explicit allow decision may enter // the crawl queue. Transient failures retain pending work; unreadable content // is unverified, and a positive policy match is rejected. -func (s *Server) prevalidate(ctx context.Context, raw string) (status, reason string) { +func (s *Server) prevalidate(ctx context.Context, raw string) (status, reason, approvedContentHash string) { if _, err := NormalizeURL(raw); err != nil { - return "rejected", "URL is not an eligible public webpage." + return "rejected", "URL is not an eligible public webpage.", "" } allowed, delay, err := s.moderationRobots.Allowed(ctx, raw) if err != nil { - return "pending", "Waiting to check the webpage." + return "pending", "Waiting to check the webpage.", "" } if !allowed { - return "unverified", "The website does not permit automated page checks." + return "unverified", "The website does not permit automated page checks.", "" } if delay > 0 { if delay > 15*time.Second { - return "unverified", "The website requires a longer crawl delay than validation supports." + return "unverified", "The website requires a longer crawl delay than validation supports.", "" } timer := time.NewTimer(delay) defer timer.Stop() select { case <-ctx.Done(): - return "pending", "Validation interrupted." + return "pending", "Validation interrupted.", "" case <-timer.C: } } req, err := http.NewRequestWithContext(ctx, "GET", raw, nil) if err != nil { - return "unverified", "The webpage could not be checked." + return "unverified", "The webpage could not be checked.", "" } req.Header.Set("User-Agent", "Cosift-Community/1.0") - req.Header.Set("Accept", "text/html, application/xhtml+xml, text/plain") + req.Header.Set("Accept", "text/html, application/xhtml+xml") res, err := s.pageClient.Do(req) if err != nil { - return "pending", "The webpage could not be reached for validation." + return "pending", "The webpage could not be reached for validation.", "" } defer res.Body.Close() if res.StatusCode == 429 || res.StatusCode >= 500 { - return "pending", "The website is temporarily unavailable for validation." + return "pending", "The website is temporarily unavailable for validation.", "" } if res.StatusCode != 200 { - return "unverified", "The webpage is unavailable or requires a login." + return "unverified", "The webpage is unavailable or requires a login.", "" } finalURL := res.Request.URL.String() if _, err := NormalizeURL(finalURL); err != nil { - return "rejected", "The destination URL is not eligible." + return "rejected", "The destination URL is not eligible.", "" } body, err := io.ReadAll(io.LimitReader(res.Body, (2<<20)+1)) if err != nil { - return "pending", "The webpage could not be read." + return "pending", "The webpage could not be read.", "" } if len(body) > 2<<20 { - return "unverified", "The webpage exceeds the validation size limit." + return "unverified", "The webpage exceeds the validation size limit.", "" } kind, _, _ := mime.ParseMediaType(res.Header.Get("Content-Type")) if kind == "" { @@ -121,27 +121,25 @@ func (s *Server) prevalidate(ctx context.Context, raw string) (status, reason st case "text/html", "application/xhtml+xml": parsed, err := crawler.Parse(body, finalURL) if err != nil { - return "unverified", "The webpage could not be interpreted." + return "unverified", "The webpage could not be interpreted.", "" } doc.Title = parsed.Title doc.Text = parsed.Text doc.Signals = pageSignals(body) - case "text/plain": - doc.Text = string(body) default: - return "unverified", "Only readable webpages can be checked; media and downloads are not accepted." + return "unverified", "Only readable HTML webpages can be checked; plain text, media, and downloads are not accepted.", "" } if adultfilter.IsAdult(doc.Title, doc.Text+" "+doc.Signals, finalURL) || strings.Contains(doc.Signals, "COSIFT_EXPLICIT_RATING") { - return "rejected", "Explicit adult content is not accepted." + return "rejected", "Explicit adult content is not accepted.", "" } if len(strings.TrimSpace(doc.Text)) < 80 { - return "unverified", "Not enough readable text to validate this webpage." + return "unverified", "Not enough readable text to validate this webpage.", "" } if len(doc.Text) > 32000 || len(doc.Title) > 1000 || len(doc.Signals) > 4000 { - return "unverified", "The webpage contains more content than can be fully checked in one validation." + return "unverified", "The webpage contains more content than can be fully checked in one validation.", "" } if status, reason := ObviousQualityProblem(doc); status != "" { - return status, reason + return status, reason, "" } b, _ := json.Marshal(doc) checkReq, _ := http.NewRequestWithContext(ctx, "POST", s.cfg.Backend+"/admin/community-moderate", bytes.NewReader(b)) @@ -151,23 +149,23 @@ func (s *Server) prevalidate(ctx context.Context, raw string) (status, reason st client.Timeout = 60 * time.Second checkRes, err := client.Do(checkReq) if err != nil { - return "pending", "Waiting for content safety checks." + return "pending", "Waiting for content safety checks.", "" } defer checkRes.Body.Close() var verdict ModerationVerdict decision := json.NewDecoder(io.LimitReader(checkRes.Body, 4096)) decision.DisallowUnknownFields() if checkRes.StatusCode != 200 || decision.Decode(&verdict) != nil || decision.Decode(new(any)) != io.EOF || !ValidVerdict(verdict) { - return "pending", "Waiting for a valid content safety decision." + return "pending", "Waiting for a valid content safety decision.", "" } switch verdict.Decision { case "allow": - return "allowed", "Content checks passed." + return "allowed", "Content checks passed.", crawler.ApprovedContentHash(doc.Title, doc.Text) case "uncertain": - return "unverified", "This webpage could not be confidently validated." + return "unverified", "This webpage could not be confidently validated.", "" default: labels := map[string]string{"adult": "Explicit adult content", "malware": "Malware distribution or malicious instructions", "phishing": "Phishing or credential theft", "graphic_violence": "Graphic violence or violent abuse", "extremist_promotion": "Extremist promotion or recruitment", "illegal_harm": "Promotion of illegal harm", "spam": "Spam or search manipulation", "low_quality": "Garbage or content without useful information"} - return "rejected", labels[verdict.Category] + " is not accepted." + return "rejected", labels[verdict.Category] + " is not accepted.", "" } } diff --git a/internal/community/moderation_binding_test.go b/internal/community/moderation_binding_test.go new file mode 100644 index 0000000..ad4656c --- /dev/null +++ b/internal/community/moderation_binding_test.go @@ -0,0 +1,116 @@ +package community + +import ( + "context" + "encoding/json" + "io" + "net/http" + "strings" + "testing" + + "github.com/pilot-protocol/cosift/internal/crawler" +) + +func TestDispatchBindsFreshModerationToEachDelivery(t *testing.T) { + for _, local := range []bool{false, true} { + name := "url" + if local { + name = "local artifact" + } + t.Run(name, func(t *testing.T) { + var approved []string + deliveries := 0 + s := testServer(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/admin/community-moderate": + var doc ModerationDocument + if err := json.NewDecoder(r.Body).Decode(&doc); err != nil { + t.Error(err) + } + approved = append(approved, crawler.ApprovedContentHash(doc.Title, doc.Text)) + io.WriteString(w, `{"decision":"allow","category":"safe"}`) + case "/admin/community-enqueue": + deliveries++ + var payload struct { + ApprovedContentHash string `json:"approved_content_hash"` + Artifact *crawler.LocalArtifact `json:"artifact"` + } + if err := json.NewDecoder(r.Body).Decode(&payload); err != nil { + t.Error(err) + } + if len(approved) != deliveries || payload.ApprovedContentHash != approved[deliveries-1] || len(payload.ApprovedContentHash) != 64 { + t.Errorf("delivery does not bind latest moderation: %+v", payload) + } + if (payload.Artifact != nil) != local { + t.Error("artifact delivery changed") + } + if deliveries == 1 { + w.WriteHeader(503) + } else { + w.WriteHeader(422) + } + default: + t.Errorf("unexpected backend request %s", r.URL.Path) + } + })) + cookie := account(t, s, "binding@example.com") + first := `First technical guide
This educational page explains programming tools, reliable systems, and defensive software engineering. It contains useful public documentation for developers.
` + second := `Updated technical guide
This revised educational reference describes compiler ownership checks, reliable memory management, and defensive programming techniques for developers building software.
` + setTestPage(s, first) + body := map[string]any{"urls": []string{"https://example.com/guide"}} + if local { + parsed, err := crawler.Parse([]byte(first), "https://example.com/guide") + if err != nil { + t.Fatal(err) + } + body = map[string]any{"artifacts": []any{map[string]any{"url": "https://example.com/guide", "title": parsed.Title, "text": parsed.Text, "model": "test", "chunks": []any{map[string]any{"text": parsed.Text, "embedding": []float32{1, 2}}}}}} + } + expect(t, request(t, s, "POST", "/api/submissions", body, cookie), 202) + if err := s.dispatch(context.Background()); err != nil { + t.Fatal(err) + } + setTestPage(s, second) + if _, err := s.db.Exec(`UPDATE submissions SET next_attempt=0`); err != nil { + t.Fatal(err) + } + if err := s.dispatch(context.Background()); err != nil { + t.Fatal(err) + } + if len(approved) != 2 || approved[0] == approved[1] { + t.Fatalf("approval was reused across changed content: %v", approved) + } + var status string + if err := s.db.QueryRow(`SELECT status FROM submissions`).Scan(&status); err != nil || status != "unverified" { + t.Fatalf("rejected approval remains retryable: %s %v", status, err) + } + if err := s.dispatch(context.Background()); err != nil { + t.Fatal(err) + } + if deliveries != 2 { + t.Fatal("permanent mismatch retried") + } + var credits int + if err := s.db.QueryRow(`SELECT count(*) FROM credit_ledger`).Scan(&credits); err != nil || credits != 0 { + t.Fatalf("failed content earned credits: %d %v", credits, err) + } + }) + } +} + +func TestPlainTextContributionIsUnverified(t *testing.T) { + s := testServer(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + t.Errorf("unsupported content reached backend %s", r.URL.Path) + })) + s.pageClient = &http.Client{Transport: pageTransport(func(r *http.Request) (*http.Response, error) { + body := "This plain text is readable and educational, but the guarded contribution crawler only supports HTML documents. It must not enter an endless delivery retry loop." + if r.URL.Path == "/robots.txt" { + body = "User-agent: *\nAllow: /\n" + } + return &http.Response{StatusCode: 200, Header: http.Header{"Content-Type": []string{"text/plain"}}, Body: io.NopCloser(strings.NewReader(body)), Request: r}, nil + })} + s.moderationRobots = crawler.NewRobots(s.pageClient, "Cosift-Community/1.0") + status, reason, hash := s.prevalidate(context.Background(), "https://example.com/readme") + if status != "unverified" || !strings.Contains(reason, "HTML") || hash != "" { + t.Fatalf("unsupported plaintext approved: %s %s %s", status, reason, hash) + } +} diff --git a/internal/community/server.go b/internal/community/server.go index e377f82..65089aa 100644 --- a/internal/community/server.go +++ b/internal/community/server.go @@ -789,7 +789,10 @@ func (s *Server) dispatch(ctx context.Context) error { if ctx.Err() != nil { return ctx.Err() } - status, reason := s.prevalidate(ctx, j.url) + status, reason, approvedContentHash := s.prevalidate(ctx, j.url) + if status == "allowed" && approvedContentHash == "" { + status, reason = "pending", "Waiting for a content-bound safety decision." + } if status != "allowed" { // Inconclusive/transient checks never fall through to enqueue. delay := time.Duration(1<= 200 && res.StatusCode < 300 if !ok { log.Printf("community: contribution delivery returned HTTP %d", res.StatusCode) From e969b6a27fc72f736a076be736dffe2994212a06 Mon Sep 17 00:00:00 2001 From: Teodor Calin Date: Thu, 17 Sep 2026 21:35:09 +0300 Subject: [PATCH 04/22] Add monthly credit display, agent setup navigation and page analytics --- internal/community/web/app.js | 41 +++++++++++++++++++++--- internal/community/webtests/app.test.cjs | 27 ++++++++++++++++ 2 files changed, 64 insertions(+), 4 deletions(-) diff --git a/internal/community/web/app.js b/internal/community/web/app.js index 7b990cc..628b9dd 100644 --- a/internal/community/web/app.js +++ b/internal/community/web/app.js @@ -32,6 +32,7 @@ function resetAccount(nextUser = null) { $("saved-count").textContent = "0"; $("credit-balance").textContent = ""; $("credit-balance").hidden = true; + $("monthly-credits").hidden = true; $("buy-credits").hidden = true; $("buy-credits").disabled = false; $("payment-info").hidden = true; @@ -262,12 +263,18 @@ $("skip-interests").onclick = async () => { }; async function refreshCredits() { $("credit-balance").hidden = !user; + $("monthly-credits").hidden = true; $("buy-credits").hidden = true; $("payment-info").hidden = true; if (user) { const c = await api("credits"); $("credit-balance").textContent = - `${c.balance} credits · 1 per extra request`; + `${Number(c.balance).toLocaleString()} credits available`; + if (c.monthly) { + $("monthly-credits").hidden = false; + $("credit-month").textContent = `This month · ${c.monthly.month} (UTC)`; + for (const name of ["earned", "purchased", "spent"]) $("month-" + name).textContent = Number(c.monthly[name] || 0).toLocaleString(); + } if (c.payments_enabled) { const pack = c.credit_pack; const price = new Intl.NumberFormat("en-US", {style: "currency", currency: pack.currency}).format(pack.amount_cents / 100); @@ -361,8 +368,9 @@ function suggestions() { } } async function view(name) { + if (name === "contribute" && !user) { showScreen("auth"); notify("Sign in to contribute webpages and earn credits."); return; } if (name === "shared" && (!sharedAuth || !user)) return; - for (const value of ["search", "saved", "contribute", "shared"]) + for (const value of ["search", "saved", "contribute", "shared", "connect"]) $("view-" + value).hidden = value !== name; document.querySelectorAll("nav [data-view]").forEach((button) => { button.classList.toggle("active", button.dataset.view === name); @@ -393,6 +401,7 @@ document .forEach((button) => (button.onclick = () => view(button.dataset.view))); $("edit-interests").onclick = () => (user ? onboarding() : showScreen("auth")); $("guest-signup").onclick = () => showScreen("auth"); +$("entry-connect").onclick = async () => { try { await enter(); await view("connect"); } catch (e) { notify(e.message, true); } }; $("continue-guest").onclick = () => { if (authBusy) return; return enter().catch((e) => notify(e.message, true)); @@ -412,7 +421,7 @@ function renderGuestAllowance() { "m " + String(seconds % 60).padStart(2, "0") + "s." - : "One request available across Search, Research, Answer, and contributions."; + : "One request available across Search, Research, and Answer. Sign in to contribute."; } setInterval(renderGuestAllowance, 1000); $("logout").onclick = async () => { @@ -617,6 +626,7 @@ async function refreshSaved() { } $("contribution-form").onsubmit = (event) => { event.preventDefault(); + if (!user) { showScreen("auth"); notify("Sign in to contribute webpages and earn credits."); return; } busy(event.target, async () => { const text = $("urls").value.trim(), file = $("csv").files[0]; @@ -661,7 +671,7 @@ async function refreshContributions() { $("contribution-list").replaceChildren( el( "div", - "Sign in to keep a personal contribution history. Guest submissions are saved for crawling without an account.", + "Sign in to contribute public webpages and track your contributions.", "empty-state", ), ); @@ -923,3 +933,26 @@ $("auth-restart").onclick = () => { $("auth-restart").hidden = true; $("auth-submit").textContent = "Email me a code →"; }; + +// The application sends only sanitized pageviews, with no search/account payload. +// Disable Enhanced Measurement in the GA stream for a pageview-only setup. +async function loadAnalytics() { + if (typeof window === "undefined") return; + try { + const response = await fetch("/api/analytics", {credentials: "same-origin"}); + if (!response.ok) return; + const config = await response.json(); + if (!/^G-[A-Z0-9]+$/.test(config.measurement_id || "")) return; + window.dataLayer = window.dataLayer || []; + const gtag = function () { window.dataLayer.push(arguments); }; + window.gtag = gtag; + gtag("js", new Date()); + gtag("config", config.measurement_id, {send_page_view: false, allow_google_signals: false, allow_ad_personalization_signals: false, page_location: location.origin + location.pathname, page_referrer: ""}); + gtag("event", "page_view", {page_location: location.origin + location.pathname, page_title: "Cosift", page_referrer: ""}); + const script = document.createElement("script"); + script.async = true; + script.src = "https://www.googletagmanager.com/gtag/js?id=" + config.measurement_id; + document.head.append(script); + } catch (_) { /* Analytics failure must not block the app. */ } +} +loadAnalytics(); diff --git a/internal/community/webtests/app.test.cjs b/internal/community/webtests/app.test.cjs index f27aaea..bb7d33f 100644 --- a/internal/community/webtests/app.test.cjs +++ b/internal/community/webtests/app.test.cjs @@ -131,3 +131,30 @@ test('email-code login waits for verification and prevents switching during an a assert.equal(form.elements.email.readOnly,false); assert.equal(a.get('code-field').hidden,true); }); +test('guests are sent to login before viewing or submitting contributions', async () => { + const a = await app(); a.run('user=null'); + await a.run('view("contribute")'); + assert.equal(a.get('auth').hidden,false); + a.get('contribution-form').onsubmit({preventDefault(){},target:a.get('contribution-form')}); + assert.equal(a.pending.some(p=>p.url==='/api/submissions'),false); + assert.match(a.get('notice').textContent,/Sign in/); +}); +test('monthly credits render actual totals and clear on account reset', async () => { + const a=await app(); const refresh=a.run('refreshCredits()'); await tick(); + a.respond('credits',{balance:42,monthly:{month:'2026-09',earned:10,purchased:50,spent:18}}); await refresh; + assert.equal(a.get('monthly-credits').hidden,false); + assert.equal(a.get('month-earned').textContent,'10'); + assert.equal(a.get('month-spent').textContent,'18'); + assert.match(a.get('credit-month').textContent,/2026-09/); + a.run('resetAccount()'); assert.equal(a.get('monthly-credits').hidden,true); +}); +test('analytics emits a pageview without query strings or account fields', async () => { + const a=await app(); a.context.window={}; a.context.document.head=node(); + Object.assign(a.context.location,{origin:'https://cosift.example',pathname:'/login',search:'?email=private@example.com&token=secret'}); + const loading=a.run('loadAnalytics()'); await tick(); + a.respond('analytics',{measurement_id:'G-XVRJ3595D1'}); await loading; + const events=JSON.stringify(a.context.window.dataLayer); + assert.match(events,/page_view/); assert.match(events,/https:\/\/cosift.example\/login/); + assert.doesNotMatch(events,/private@example|secret|Alice|user_id/); + assert.equal(a.context.document.head.children[0].src,'https://www.googletagmanager.com/gtag/js?id=G-XVRJ3595D1'); +}); From a4d63222cc80c50d9421b8de0c36af10f6e52f6f Mon Sep 17 00:00:00 2001 From: Teodor Calin Date: Thu, 17 Sep 2026 21:36:37 +0300 Subject: [PATCH 05/22] Bind contribution indexing and receipts to approved content --- cmd/cosift/community_receipt.go | 45 ++++++++-- cmd/cosift/community_receipt_test.go | 69 +++++++++++++-- cmd/cosift/community_test.go | 4 +- cmd/cosift/serve_crawl.go | 13 ++- cmd/cosift/serve_setup.go | 2 +- internal/crawler/artifact_test.go | 2 +- internal/crawler/contribution.go | 22 ++++- .../crawler/contribution_approval_test.go | 84 +++++++++++++++++++ internal/crawler/crawler.go | 19 ++++- 9 files changed, 234 insertions(+), 26 deletions(-) create mode 100644 internal/crawler/contribution_approval_test.go diff --git a/cmd/cosift/community_receipt.go b/cmd/cosift/community_receipt.go index 60b1bff..6b484cd 100644 --- a/cmd/cosift/community_receipt.go +++ b/cmd/cosift/community_receipt.go @@ -16,17 +16,21 @@ import ( var errCommunityReceiptConflict = errors.New("submission id belongs to another payload") type communityReceiptRecord struct { - PayloadHash string `json:"payload_hash"` - Eligible bool `json:"eligible"` - Receipt *crawler.ContributionReceipt `json:"receipt,omitempty"` + PayloadHash string `json:"payload_hash"` + ApprovedContentHash string `json:"approved_content_hash,omitempty"` + Eligible bool `json:"eligible"` + Receipt *crawler.ContributionReceipt `json:"receipt,omitempty"` } // Journal intent before indexing and the receipt before returning HTTP success. // A lost response, failed reward write, or restart must replay the same novelty // decision. Keys live in Pebble's metadata family and are included in backups. -func (s *pebbleHTTP) fetchCommunityReceipt(ctx context.Context, id, raw string, artifact *crawler.LocalArtifact) (crawler.ContributionReceipt, error) { +func (s *pebbleHTTP) fetchCommunityReceipt(ctx context.Context, id, raw string, artifact *crawler.LocalArtifact, approvedHash string) (crawler.ContributionReceipt, error) { + if !crawler.ValidApprovedContentHash(approvedHash) { + return crawler.ContributionReceipt{}, fmt.Errorf("%w: approved content hash is required", crawler.ErrContributionRejected) + } if id == "" { - return s.crawlCommunityFetch(ctx, raw, artifact) + return s.crawlCommunityFetch(ctx, raw, artifact, approvedHash) } // older trusted portals if s.store == nil { return crawler.ContributionReceipt{}, fmt.Errorf("receipt store unavailable") @@ -41,11 +45,12 @@ func (s *pebbleHTTP) fetchCommunityReceipt(ctx context.Context, id, raw string, Artifact *crawler.LocalArtifact }{raw, artifact}) hash := sha256.Sum256(payload) - record := communityReceiptRecord{PayloadHash: hex.EncodeToString(hash[:])} + record := communityReceiptRecord{PayloadHash: hex.EncodeToString(hash[:]), ApprovedContentHash: approvedHash} key := []byte("mcommunity_receipt:" + id) db := s.store.DB() encoded, closer, err := db.Get(key) if err == nil { + record = communityReceiptRecord{} err = json.Unmarshal(encoded, &record) closer.Close() if err != nil { @@ -54,6 +59,32 @@ func (s *pebbleHTTP) fetchCommunityReceipt(ctx context.Context, id, raw string, if record.PayloadHash != hex.EncodeToString(hash[:]) { return crawler.ContributionReceipt{}, errCommunityReceiptConflict } + if record.ApprovedContentHash == "" { + // Upgrade a legacy receipt only when its indexed content is the + // exact document approved now. Never grandfather unbound content. + if record.Receipt != nil { + canon, canonicalErr := crawler.ContributionURL(raw) + if canonicalErr != nil { + return crawler.ContributionReceipt{}, canonicalErr + } + doc, docErr := s.store.GetDocByURL(ctx, canon) + if docErr != nil || doc == nil || crawler.ApprovedContentHash(doc.Title, doc.Text) != approvedHash { + return crawler.ContributionReceipt{}, fmt.Errorf("%w: legacy receipt content is not approved", crawler.ErrContributionRejected) + } + textHash := sha256.Sum256([]byte(doc.Text)) + if record.Receipt.ContentHash != hex.EncodeToString(textHash[:]) { + return crawler.ContributionReceipt{}, fmt.Errorf("%w: legacy receipt content changed", crawler.ErrContributionRejected) + } + } + record.ApprovedContentHash = approvedHash + data, _ := json.Marshal(record) + if err := db.Set(key, data, pebble.Sync); err != nil { + return crawler.ContributionReceipt{}, err + } + } + if record.ApprovedContentHash != approvedHash { + return crawler.ContributionReceipt{}, fmt.Errorf("%w: approval differs from recorded submission", crawler.ErrContributionRejected) + } if record.Receipt != nil { return *record.Receipt, nil } @@ -74,7 +105,7 @@ func (s *pebbleHTTP) fetchCommunityReceipt(ctx context.Context, id, raw string, } else { return crawler.ContributionReceipt{}, err } - receipt, err := s.crawlCommunityFetch(ctx, raw, artifact) + receipt, err := s.crawlCommunityFetch(ctx, raw, artifact, approvedHash) if err != nil { return crawler.ContributionReceipt{}, err } diff --git a/cmd/cosift/community_receipt_test.go b/cmd/cosift/community_receipt_test.go index a33cc7c..b7986fd 100644 --- a/cmd/cosift/community_receipt_test.go +++ b/cmd/cosift/community_receipt_test.go @@ -2,7 +2,11 @@ package main import ( "context" + "crypto/sha256" + "encoding/hex" "encoding/json" + "errors" + "github.com/cockroachdb/pebble" "net/http/httptest" "strings" "testing" @@ -20,7 +24,7 @@ func TestCommunityReceiptSurvivesLostResponseAndRestart(t *testing.T) { } calls := 0 setup := func() *pebbleHTTP { - s := &pebbleHTTP{store: db, cluster: config.Cluster{PeerAuthToken: "secret"}, crawlCommunityFetch: func(context.Context, string, *crawler.LocalArtifact) (crawler.ContributionReceipt, error) { + s := &pebbleHTTP{store: db, cluster: config.Cluster{PeerAuthToken: "secret"}, crawlCommunityFetch: func(context.Context, string, *crawler.LocalArtifact, string) (crawler.ContributionReceipt, error) { calls++ // Simulate a recrawl: only the first delivery can report novelty. return crawler.ContributionReceipt{Indexed: true, Novel: calls == 1, ContentHash: strings.Repeat("a", 64)}, nil @@ -29,7 +33,7 @@ func TestCommunityReceiptSurvivesLostResponseAndRestart(t *testing.T) { return s } invoke := func(s *pebbleHTTP, raw string) *httptest.ResponseRecorder { - r := httptest.NewRequest("POST", "/admin/community-enqueue", strings.NewReader(`{"submission_id":"stable-submission-123","url":"`+raw+`"}`)) + r := httptest.NewRequest("POST", "/admin/community-enqueue", strings.NewReader(`{"submission_id":"stable-submission-123","url":"`+raw+`","approved_content_hash":"`+strings.Repeat("a", 64)+`"}`)) r.Header.Set("Authorization", "Bearer secret") w := httptest.NewRecorder() s.handleCommunityEnqueue(w, r) @@ -65,14 +69,14 @@ func TestCommunityReceiptResumesInterruptedIndexing(t *testing.T) { t.Fatal(err) } ctx := context.Background() - s := &pebbleHTTP{store: db, crawlCommunityFetch: func(ctx context.Context, raw string, a *crawler.LocalArtifact) (crawler.ContributionReceipt, error) { + s := &pebbleHTTP{store: db, crawlCommunityFetch: func(ctx context.Context, raw string, a *crawler.LocalArtifact, approvedHash string) (crawler.ContributionReceipt, error) { _, err := db.UpsertDocument(ctx, &store.Document{URL: raw, Title: "Guide", Text: "Useful content"}) if err != nil { t.Fatal(err) } return crawler.ContributionReceipt{}, context.DeadlineExceeded }} - if _, err := s.fetchCommunityReceipt(ctx, "interrupted-job-123", "https://example.com/guide", nil); err == nil { + if _, err := s.fetchCommunityReceipt(ctx, "interrupted-job-123", "https://example.com/guide", nil, strings.Repeat("a", 64)); err == nil { t.Fatal("expected interruption") } db.Close() @@ -81,16 +85,67 @@ func TestCommunityReceiptResumesInterruptedIndexing(t *testing.T) { t.Fatal(err) } defer db.Close() - s = &pebbleHTTP{store: db, crawlCommunityFetch: func(context.Context, string, *crawler.LocalArtifact) (crawler.ContributionReceipt, error) { + s = &pebbleHTTP{store: db, crawlCommunityFetch: func(context.Context, string, *crawler.LocalArtifact, string) (crawler.ContributionReceipt, error) { return crawler.ContributionReceipt{Indexed: true, Novel: false, ContentHash: strings.Repeat("b", 64)}, nil }} - receipt, err := s.fetchCommunityReceipt(ctx, "interrupted-job-123", "https://example.com/guide", nil) + receipt, err := s.fetchCommunityReceipt(ctx, "interrupted-job-123", "https://example.com/guide", nil, strings.Repeat("a", 64)) if err != nil || !receipt.Novel { t.Fatalf("lost original eligibility: %+v %v", receipt, err) } // Tracking parameters must not turn an existing document into a paid contribution. - receipt, err = s.fetchCommunityReceipt(ctx, "different-job-123", "https://example.com/guide?utm_source=credits", nil) + receipt, err = s.fetchCommunityReceipt(ctx, "different-job-123", "https://example.com/guide?utm_source=credits", nil, strings.Repeat("a", 64)) if err != nil || receipt.Novel { t.Fatalf("existing canonical document rewarded: %+v %v", receipt, err) } } + +func TestCommunityReceiptApprovalBindingAndLegacyUpgrade(t *testing.T) { + db, err := store.OpenPebble(t.TempDir()) + if err != nil { + t.Fatal(err) + } + defer db.Close() + ctx := context.Background() + raw := "https://example.com/approved" + title, text := "Guide", "Safe approved content" + if _, err := db.UpsertDocument(ctx, &store.Document{URL: raw, Title: title, Text: text}); err != nil { + t.Fatal(err) + } + payload, _ := json.Marshal(struct { + URL string + Artifact *crawler.LocalArtifact + }{raw, nil}) + ph := sha256.Sum256(payload) + th := sha256.Sum256([]byte(text)) + record := communityReceiptRecord{PayloadHash: hex.EncodeToString(ph[:]), Eligible: true, Receipt: &crawler.ContributionReceipt{Indexed: true, Novel: true, ContentHash: hex.EncodeToString(th[:])}} + encoded, _ := json.Marshal(record) + key := []byte("mcommunity_receipt:legacy-approval-job") + if err := db.DB().Set(key, encoded, pebble.Sync); err != nil { + t.Fatal(err) + } + calls := 0 + s := &pebbleHTTP{store: db, crawlCommunityFetch: func(context.Context, string, *crawler.LocalArtifact, string) (crawler.ContributionReceipt, error) { + calls++ + return crawler.ContributionReceipt{}, nil + }} + if _, err := s.fetchCommunityReceipt(ctx, "legacy-approval-job", raw, nil, crawler.ApprovedContentHash(title, "Different")); !errors.Is(err, crawler.ErrContributionRejected) { + t.Fatalf("legacy receipt bypassed approval: %v", err) + } + approved := crawler.ApprovedContentHash(title, text) + receipt, err := s.fetchCommunityReceipt(ctx, "legacy-approval-job", raw, nil, approved) + if err != nil || !receipt.Novel || calls != 0 { + t.Fatalf("legacy unchanged replay failed: %+v %v calls=%d", receipt, err, calls) + } + encoded, closer, err := db.DB().Get(key) + if err != nil { + t.Fatal(err) + } + defer closer.Close() + var upgraded communityReceiptRecord + if err := json.Unmarshal(encoded, &upgraded); err != nil || upgraded.ApprovedContentHash != approved { + t.Fatal("legacy receipt approval not persisted") + } + if _, err := s.fetchCommunityReceipt(ctx, "legacy-approval-job", raw, nil, crawler.ApprovedContentHash(title, "Different")); !errors.Is(err, crawler.ErrContributionRejected) { + t.Fatalf("bound receipt bypassed approval: %v", err) + } +} diff --git a/cmd/cosift/community_test.go b/cmd/cosift/community_test.go index 563b8e2..2529317 100644 --- a/cmd/cosift/community_test.go +++ b/cmd/cosift/community_test.go @@ -18,7 +18,7 @@ import ( func TestCommunityEnqueueRequiresGuardAndAuth(t *testing.T) { called := 0 - s := &pebbleHTTP{cluster: config.Cluster{PeerAuthToken: "secret"}, crawlCommunityFetch: func(ctx context.Context, raw string, artifact *crawler.LocalArtifact) (crawler.ContributionReceipt, error) { + s := &pebbleHTTP{cluster: config.Cluster{PeerAuthToken: "secret"}, crawlCommunityFetch: func(ctx context.Context, raw string, artifact *crawler.LocalArtifact, approvedHash string) (crawler.ContributionReceipt, error) { called++ if raw != "https://example.com/guide" { t.Errorf("bad contribution %s", raw) @@ -26,7 +26,7 @@ func TestCommunityEnqueueRequiresGuardAndAuth(t *testing.T) { return crawler.ContributionReceipt{Indexed: true, Novel: true}, nil }} call := func(token, url string) int { - r := httptest.NewRequest("POST", "/admin/community-enqueue", strings.NewReader(`{"url":"`+url+`"}`)) + r := httptest.NewRequest("POST", "/admin/community-enqueue", strings.NewReader(`{"url":"`+url+`","approved_content_hash":"`+strings.Repeat("a", 64)+`"}`)) r.Header.Set("Authorization", "Bearer "+token) w := httptest.NewRecorder() s.handleCommunityEnqueue(w, r) diff --git a/cmd/cosift/serve_crawl.go b/cmd/cosift/serve_crawl.go index 6468d2d..61a90c4 100644 --- a/cmd/cosift/serve_crawl.go +++ b/cmd/cosift/serve_crawl.go @@ -42,15 +42,20 @@ func (s *pebbleHTTP) handleCommunityEnqueue(w http.ResponseWriter, r *http.Reque return } var req struct { - SubmissionID string `json:"submission_id,omitempty"` - URL string `json:"url"` - Artifact *crawler.LocalArtifact `json:"artifact,omitempty"` + SubmissionID string `json:"submission_id,omitempty"` + URL string `json:"url"` + Artifact *crawler.LocalArtifact `json:"artifact,omitempty"` + ApprovedContentHash string `json:"approved_content_hash"` } r.Body = http.MaxBytesReader(w, r.Body, 1<<20) if json.NewDecoder(r.Body).Decode(&req) != nil { writeProblem(w, http.StatusBadRequest, "expected a webpage URL") return } + if !crawler.ValidApprovedContentHash(req.ApprovedContentHash) { + writeProblem(w, http.StatusUnprocessableEntity, "approved content hash is required") + return + } if req.SubmissionID != "" { if len(req.SubmissionID) < 16 || len(req.SubmissionID) > 64 || strings.IndexFunc(req.SubmissionID, func(r rune) bool { return !(r >= 'a' && r <= 'z' || r >= 'A' && r <= 'Z' || r >= '0' && r <= '9' || r == '_' || r == '-') @@ -67,7 +72,7 @@ func (s *pebbleHTTP) handleCommunityEnqueue(w http.ResponseWriter, r *http.Reque liftWriteDeadline(w) ctx, cancel := context.WithTimeout(r.Context(), 2*time.Minute) defer cancel() - receipt, err := s.fetchCommunityReceipt(ctx, req.SubmissionID, u, req.Artifact) + receipt, err := s.fetchCommunityReceipt(ctx, req.SubmissionID, u, req.Artifact, req.ApprovedContentHash) if err != nil { if errors.Is(err, errCommunityReceiptConflict) { writeProblem(w, http.StatusConflict, "submission id belongs to another payload") diff --git a/cmd/cosift/serve_setup.go b/cmd/cosift/serve_setup.go index 10f492c..08a676c 100644 --- a/cmd/cosift/serve_setup.go +++ b/cmd/cosift/serve_setup.go @@ -1268,7 +1268,7 @@ type pebbleHTTP struct { crawlSeed func(url string) error crawlPublicOnly atomic.Bool crawlCommunityReady atomic.Bool - crawlCommunityFetch func(context.Context, string, *crawler.LocalArtifact) (crawler.ContributionReceipt, error) + crawlCommunityFetch func(context.Context, string, *crawler.LocalArtifact, string) (crawler.ContributionReceipt, error) // crawlSeedSitemap wraps Crawler.SeedSitemap so the /admin/ // sitemap-import endpoint can push sitemap URLs into the live frontier. crawlSeedSitemap func(ctx context.Context, url string) (int, error) diff --git a/internal/crawler/artifact_test.go b/internal/crawler/artifact_test.go index 6b6eb30..cad1453 100644 --- a/internal/crawler/artifact_test.go +++ b/internal/crawler/artifact_test.go @@ -66,7 +66,7 @@ func TestContributionDoesNotUseBulkRemoteFetcher(t *testing.T) { } defer db.Close() c := NewWithBackend(config.Crawler{RemoteFetcherURL: remote.URL, AutoSitemap: true, FilterAdult: false}, db, index.NewBM25(db)) - _, err = c.FetchContribution(context.Background(), "http://127.0.0.1/private", nil) + _, err = c.FetchContribution(context.Background(), "http://127.0.0.1/private", nil, ApprovedContentHash("Guide", "Body")) if err == nil { t.Fatal("private URL accepted") } diff --git a/internal/crawler/contribution.go b/internal/crawler/contribution.go index 803b03b..36d9d1d 100644 --- a/internal/crawler/contribution.go +++ b/internal/crawler/contribution.go @@ -18,10 +18,27 @@ type ContributionReceipt struct { ContentHash string `json:"content_hash"` } +// ApprovedContentHash binds a moderation decision to the exact parsed document. +func ApprovedContentHash(title, text string) string { + sum := sha256.Sum256([]byte(title + "\x00" + text)) + return hex.EncodeToString(sum[:]) +} + +func ValidApprovedContentHash(value string) bool { + raw, err := hex.DecodeString(value) + return err == nil && len(raw) == sha256.Size && hex.EncodeToString(raw) == value +} + +type approvedContentKey struct{} + // ContributionURL returns the document key used when indexing a contribution. func ContributionURL(raw string) (string, error) { return canonicalize(raw) } -func (c *Crawler) FetchContribution(ctx context.Context, raw string, artifact *LocalArtifact) (ContributionReceipt, error) { +func (c *Crawler) FetchContribution(ctx context.Context, raw string, artifact *LocalArtifact, approvedHash string) (ContributionReceipt, error) { + if !ValidApprovedContentHash(approvedHash) { + return ContributionReceipt{}, fmt.Errorf("%w: approved content hash is required", ErrContributionRejected) + } + ctx = context.WithValue(ctx, approvedContentKey{}, approvedHash) canon, err := canonicalize(raw) if err != nil { return ContributionReceipt{}, err @@ -68,6 +85,9 @@ func (c *Crawler) FetchContribution(ctx context.Context, raw string, artifact *L if err != nil { return ContributionReceipt{}, err } + if ApprovedContentHash(page.Title, page.Text) != approvedHash { + return ContributionReceipt{}, fmt.Errorf("%w: webpage changed after moderation", ErrContributionRejected) + } verified, err := VerifyArtifact(ctx, artifact, page.Title, page.Text, c.embedder) if err != nil { return ContributionReceipt{}, err diff --git a/internal/crawler/contribution_approval_test.go b/internal/crawler/contribution_approval_test.go new file mode 100644 index 0000000..289c70d --- /dev/null +++ b/internal/crawler/contribution_approval_test.go @@ -0,0 +1,84 @@ +package crawler + +import ( + "context" + "errors" + "net/http" + "net/http/httptest" + "strings" + "testing" +) + +func TestContributionApprovalRejectsChangedContentBeforeIndex(t *testing.T) { + for _, existing := range []bool{false, true} { + t.Run(map[bool]string{false: "new", true: "existing"}[existing], func(t *testing.T) { + body := sampleHTML + requests := 0 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests++ + if r.Header.Get("If-None-Match") != "" { + t.Error("approved fetch used conditional cache") + } + w.Header().Set("Content-Type", "text/html") + w.Header().Set("ETag", "approved-page") + _, _ = w.Write([]byte(body)) + })) + defer srv.Close() + db := newStoreT(t) + cfg := testCrawlerCfg() + cfg.RespectRobots = false + cfg.PerHostDelayMs = 0 + cfg.DisableLinkFollowing = true + emb := &stubEmbedder{dim: 8} + c := New(cfg, db).WithEmbedder(emb) + // Local transport fixture only; production constructs a guarded client. + c.contributionOnce.Do(func() { c.contributionCrawler = c; c.contributionSlots = make(chan struct{}, 2) }) + parsed, err := Parse([]byte(sampleHTML), srv.URL) + if err != nil { + t.Fatal(err) + } + approved := ApprovedContentHash(parsed.Title, parsed.Text) + if existing { + if _, err := c.FetchContribution(context.Background(), srv.URL, nil, approved); err != nil { + t.Fatal(err) + } + } + embBefore, requestsBefore := emb.calls, requests + // Fresh-cache settings must not bypass the exact-content check. + t.Setenv("COSIFT_REFETCH_AFTER_HOURS", "999") + body = strings.ReplaceAll(sampleHTML, "Go programming", "credential harvesting and malicious spam") + if _, err := c.FetchContribution(context.Background(), srv.URL, nil, approved); !errors.Is(err, ErrContributionRejected) { + t.Fatalf("changed page accepted: %v", err) + } + if requests != requestsBefore+1 || emb.calls != embBefore { + t.Fatal("approval bypassed fetch or embedded rejected content") + } + doc, err := db.GetDocByURL(context.Background(), srv.URL) + if existing { + if err != nil || doc == nil || doc.Text != parsed.Text { + t.Fatal("rejected content replaced the indexed document") + } + } else if err == nil && doc != nil { + t.Fatal("rejected content entered the index") + } + }) + } +} + +func TestContributionApprovalRequiresHashAndFreshBody(t *testing.T) { + db := newStoreT(t) + cfg := testCrawlerCfg() + cfg.RespectRobots = false + c := New(cfg, db) + for _, value := range []string{"", "bad", strings.Repeat("A", 64)} { + if _, err := c.FetchContribution(context.Background(), "https://example.com", nil, value); !errors.Is(err, ErrContributionRejected) { + t.Fatalf("invalid approval accepted: %q %v", value, err) + } + } + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusNotModified) })) + defer srv.Close() + ctx := context.WithValue(context.Background(), approvedContentKey{}, ApprovedContentHash("title", "text")) + if err := c.FetchAndIndexNow(ctx, srv.URL); !errors.Is(err, ErrContributionRejected) { + t.Fatalf("304 bypassed approval: %v", err) + } +} diff --git a/internal/crawler/crawler.go b/internal/crawler/crawler.go index 21acbb4..70c4353 100644 --- a/internal/crawler/crawler.go +++ b/internal/crawler/crawler.go @@ -1227,6 +1227,7 @@ func (c *Crawler) FetchAndIndexNow(ctx context.Context, url string) error { } func (c *Crawler) processClaimed(ctx context.Context, item store.FrontierItem, gate *hostGate) error { + approvedHash, _ := ctx.Value(approvedContentKey{}).(string) u, _ := url.Parse(item.URL) // Prior enqueueLinks already // filters via allowedDomain, but stale frontier entries from before @@ -1289,7 +1290,7 @@ func (c *Crawler) processClaimed(ctx context.Context, item store.FrontierItem, g // revisits at the cost of missing freshly-updated content during the // staleness window. Default 0 = disabled (every revisit issues a // conditional GET, matching behavior). - if prior != nil && len(prior.ContentSHA) > 0 { + if approvedHash == "" && prior != nil && len(prior.ContentSHA) > 0 { refetchHours := 0 if v := os.Getenv("COSIFT_REFETCH_AFTER_HOURS"); v != "" { if n, err := strconv.Atoi(v); err == nil && n > 0 { @@ -1303,10 +1304,19 @@ func (c *Crawler) processClaimed(ctx context.Context, item store.FrontierItem, g } } - res, err := c.fetch(ctx, item.URL, prior) + fetchPrior := prior + if approvedHash != "" { + // A fresh full body must match moderation; cached/304 shortcuts cannot + // establish which content this contribution actually approved. + fetchPrior = nil + } + res, err := c.fetch(ctx, item.URL, fetchPrior) if err != nil { return err } + if approvedHash != "" && res.notModified { + return fmt.Errorf("%w: moderation requires a fresh response body", ErrContributionRejected) + } // 304 Not Modified: server confirmed nothing changed. Update validators // + fetched_at, skip parse / BM25 / embed. Zero body bandwidth. @@ -1349,6 +1359,9 @@ func (c *Crawler) processClaimed(ctx context.Context, item store.FrontierItem, g if perr != nil { return perr } + if approvedHash != "" && ApprovedContentHash(parsed.Title, parsed.Text) != approvedHash { + return fmt.Errorf("%w: webpage changed after moderation", ErrContributionRejected) + } if strings.TrimSpace(parsed.Text) == "" { return errors.New("empty content") } @@ -1372,7 +1385,7 @@ func (c *Crawler) processClaimed(ctx context.Context, item store.FrontierItem, g // the index work is already done. Update validators + fetched_at and exit. // Catches servers that don't send ETag/Last-Modified (so 304 isn't available). if existing := prior; existing != nil { - if bytes.Equal(existing.ContentSHA, sha[:]) { + if bytes.Equal(existing.ContentSHA, sha[:]) && (approvedHash == "" || existing.Title == parsed.Title) { existing.FetchedAt = time.Now() if res.etag != "" { existing.ETag = res.etag From 382dd08f0e7677ea46ce6fccb37df8ea1d8d012d Mon Sep 17 00:00:00 2001 From: Teodor Calin Date: Thu, 17 Sep 2026 21:37:13 +0300 Subject: [PATCH 06/22] Simplify the community interface and agent setup layout --- internal/community/web/index.html | 125 ++++++++++-------------- internal/community/web/style.css | 152 ++++++++++++++++++++++++++++++ 2 files changed, 203 insertions(+), 74 deletions(-) diff --git a/internal/community/web/index.html b/internal/community/web/index.html index ef8a120..ddff5f5 100644 --- a/internal/community/web/index.html +++ b/internal/community/web/index.html @@ -3,7 +3,7 @@ - Cosift — your corner of the web + Cosift @@ -13,24 +13,18 @@
cosift●
-

A web worth finding

-

A little curiosity.
A better internet.

+

Search the web.
Keep what matters.

- Find something useful. Keep what matters.
Share the corners of - the web you know best. + Search, research, and contribute with one account.

-
- Independent sourcesShared discovery -
-

Built for curious people. Made better together.

+

Web · CLI · Agents

-

Welcome to Cosift

-

Make yourself at home.

+

Sign in to Cosift

- One account for your searches and contributions. + Use the same email as your CLI and agents.

@@ -76,11 +71,9 @@

Make yourself at home.

-