Add community web app, local indexing CLI and contribution credits

.github/workflows/deploy.yml

@@ -4,7 +4,7 @@ name: deploy
 # this workflow never touches the production box and never holds an SSH key.
 # The box runs cosift-self-update.timer, which polls the latest GitHub Release,
 # verifies sha256 + minisign signature against a public key baked on the box,
-# snapshots, atomically swaps the binary, restarts, and health-gates with
+# atomically swaps the binary, restarts, and health-gates with
 # auto-rollback. See deploy/scripts/README.md.
 #
 # Triggers:
@@ -37,14 +37,14 @@ jobs:
         run: go vet ./...
 
       - name: Tests (race)
-        run: go test -race -timeout 5m ./...
+        run: go test -race -timeout 10m ./...
 
       - name: Smoke subset (build + serve roundtrip)
         # make smoke builds the binary, crawls, and asserts /healthz + /search.
         # Guarded with a timeout so a hung crawl can't wedge the release.
         run: timeout 300 make smoke
 
-  # (b) build — reproducible static arm64 binary, sha256, minisign signature.
+  # (b) build — static server and CLI binaries, sha256, minisign signatures.
   build:
     needs: verify
     runs-on: ubuntu-latest
@@ -70,22 +70,19 @@ jobs:
             echo "version=$(git rev-parse --short HEAD)" >> "$GITHUB_OUTPUT"
           fi
 
-      - name: Build (linux/arm64, static, trimmed)
+      - name: Build CLI and server binaries
         env:
           CGO_ENABLED: '0'
-          GOOS: linux
-          GOARCH: arm64
           VERSION: ${{ steps.ver.outputs.version }}
-        # Stamp both main.version and internal/server.Version so /healthz and
-        # /metrics report the shipped version. Matches the Makefile `build`
-        # target's ldflags exactly (kept in sync deliberately).
         run: |
-          go build -trimpath \
-            -ldflags "-s -w -X main.version=${VERSION} -X github.com/pilot-protocol/cosift/internal/server.Version=${VERSION}" \
-            -o cosift-linux-arm64 ./cmd/cosift
-
-      - name: sha256
-        run: sha256sum cosift-linux-arm64 | tee cosift-linux-arm64.sha256
+          mkdir -p dist
+          for target in linux/arm64 linux/amd64 darwin/arm64 darwin/amd64 windows/amd64; do
+            export GOOS="${target%/*}" GOARCH="${target#*/}"
+            name="cosift-${GOOS}-${GOARCH}"
+            if [ "$GOOS" = windows ]; then name="${name}.exe"; fi
+            go build -trimpath -ldflags "-s -w -X main.version=${VERSION} -X github.com/pilot-protocol/cosift/internal/server.Version=${VERSION}" -o "dist/$name" ./cmd/cosift
+            (cd dist && sha256sum "$name" > "$name.sha256")
+          done
 
       - name: Install minisign
         run: sudo apt-get update && sudo apt-get install -y minisign
@@ -104,21 +101,16 @@ jobs:
           printf '%s' "${MINISIGN_SECRET_KEY}" > minisign.key
           # -W: secret key is unencrypted (no interactive passphrase prompt).
           # Trusted comment carries the version so the box can sanity-check it.
-          minisign -S -W -s minisign.key \
-            -m cosift-linux-arm64 \
-            -t "cosift ${{ steps.ver.outputs.version }} linux/arm64"
+          for binary in dist/cosift-*; do
+            case "$binary" in *.sha256) continue;; esac
+            minisign -S -W -s minisign.key -m "$binary" -t "cosift ${{ steps.ver.outputs.version }} $(basename "$binary")"
+          done
           rm -f minisign.key
-          # Verification against the embedded public key is done on the box;
-          # here we just confirm the .minisig was produced.
-          test -f cosift-linux-arm64.minisig
 
       - uses: actions/upload-artifact@v7
         with:
           name: cosift-release-artifacts
-          path: |
-            cosift-linux-arm64
-            cosift-linux-arm64.sha256
-            cosift-linux-arm64.minisig
+          path: dist/*
           retention-days: 30
 
   # (c) release — publish the signed binary as a GitHub Release asset.
@@ -142,7 +134,4 @@ jobs:
           name: cosift ${{ github.ref_name }}
           generate_release_notes: true
           fail_on_unmatched_files: true
-          files: |
-            cosift-linux-arm64
-            cosift-linux-arm64.sha256
-            cosift-linux-arm64.minisig
+          files: cosift-*

.gitignore

@@ -5,6 +5,7 @@
 
 # Local data + runtime state
 cosift-data/
+community-data/
 cosift.db
 *.db
 *.db-wal

README.md

@@ -19,6 +19,11 @@ seed URLs  ───▶  │  crawler  →  index  →  retriever     │  ─
 
 ## Quick start
 
+For the contributor web app (guest access, email/password accounts, interest
+onboarding, Search/Research/Answer, saved requests, and checked URL/CSV contributions), see
+[Community app and CLI](docs/COMMUNITY.md). Run `cosift community` alongside a
+Pebble backend; guests get one search or submission every 30 minutes.
+
 ```bash
 # 1. Build
 go build -o cosift ./cmd/cosift

cmd/cosift/community.go

@@ -0,0 +1,209 @@
+package main
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"github.com/pilot-protocol/cosift/internal/config"
+	"io"
+	"log"
+	"net/http"
+	"net/http/cookiejar"
+	"net/url"
+	"os"
+	"strings"
+	"time"
+
+	"github.com/pilot-protocol/cosift/internal/community"
+)
+
+func runCommunity(ctx context.Context, args []string) error {
+	fs := flag.NewFlagSet("community", flag.ContinueOnError)
+	addr := fs.String("addr", "127.0.0.1:7780", "listen address")
+	publicURL := fs.String("public-url", "http://127.0.0.1:7780", "browser origin; HTTPS required outside localhost")
+	backend := fs.String("backend", "http://127.0.0.1:7777", "Cosift Pebble server origin")
+	dir := fs.String("data-dir", "./community-data", "private account database directory")
+	proxies := fs.String("trusted-proxies", "", "comma-separated proxy CIDRs allowed to supply X-Forwarded-For")
+	if err := fs.Parse(args); err != nil {
+		return err
+	}
+	if fs.NArg() != 0 {
+		return fmt.Errorf("unexpected arguments: %v", fs.Args())
+	}
+	var trusted []string
+	if *proxies != "" {
+		trusted = strings.Split(*proxies, ",")
+	}
+	s, err := community.Open(community.Config{DataDir: *dir, Backend: *backend, PublicURL: *publicURL, AdminToken: os.Getenv("COSIFT_COMMUNITY_ADMIN_TOKEN"), TrustedProxies: trusted})
+	if err != nil {
+		return err
+	}
+	defer s.Close()
+	workerCtx, cancel := context.WithCancel(ctx)
+	defer cancel()
+	done := make(chan struct{})
+	go func() { defer close(done); s.Run(workerCtx) }()
+	defer func() { cancel(); <-done }()
+	srv := &http.Server{Addr: *addr, Handler: s, ReadHeaderTimeout: 5 * time.Second, ReadTimeout: 20 * time.Second, WriteTimeout: 4 * time.Minute, IdleTimeout: 60 * time.Second, MaxHeaderBytes: 16 << 10}
+	stopped := make(chan struct{})
+	defer close(stopped)
+	go func() {
+		select {
+		case <-ctx.Done():
+			shutdownCtx, c := context.WithTimeout(context.Background(), 5*time.Second)
+			defer c()
+			_ = srv.Shutdown(shutdownCtx)
+		case <-stopped:
+		}
+	}()
+	log.Printf("community: listening on %s (public origin %s)", *addr, *publicURL)
+	if err := srv.ListenAndServe(); err != nil && err != http.ErrServerClosed {
+		return err
+	}
+	return nil
+}
+
+func runContribute(ctx context.Context, args []string) error {
+	return runContributeConfigured(ctx, nil, args)
+}
+
+func runContributeConfigured(ctx context.Context, cfg *config.Config, args []string) error {
+	fs := flag.NewFlagSet("contribute", flag.ContinueOnError)
+	server := fs.String("server", "http://127.0.0.1:7780", "community app origin")
+	email := fs.String("email", os.Getenv("COSIFT_EMAIL"), "account email (or COSIFT_EMAIL)")
+	file := fs.String("csv", "", "CSV file with webpage URLs; - reads stdin")
+	requestMode := fs.Bool("request", false, "perform a community Search, Answer or Research request")
+	query := fs.String("query", "", "query for a community request")
+	mode := fs.String("mode", "search", "search, answer or research")
+	local := fs.Bool("index-locally", false, "fetch, index and embed locally, then contribute verified artifacts (requires login and embedding config)")
+	credits := fs.Bool("credits", false, "show the authenticated account credit balance")
+	guest := fs.Bool("guest", false, "submit without login (one request per 30 minutes per IP)")
+	if err := fs.Parse(args); err != nil {
+		return err
+	}
+	u, err := url.Parse(*server)
+	if err != nil || u.Host == "" || u.User != nil || u.RawQuery != "" || u.Fragment != "" || (u.Path != "" && u.Path != "/") || (u.Scheme != "http" && u.Scheme != "https") {
+		return fmt.Errorf("server must be an http(s) origin")
+	}
+	if u.Scheme != "https" && u.Hostname() != "localhost" && u.Hostname() != "127.0.0.1" && u.Hostname() != "::1" {
+		return fmt.Errorf("use HTTPS to protect account credentials")
+	}
+	password := os.Getenv("COSIFT_PASSWORD")
+	if !*guest && ((*email == "") != (password == "")) {
+		return fmt.Errorf("set both COSIFT_EMAIL and COSIFT_PASSWORD, or use -guest")
+	}
+	values := fs.Args()
+	if *file != "" {
+		if len(values) > 0 {
+			return fmt.Errorf("use either -csv or positional URLs")
+		}
+		var reader io.Reader = os.Stdin
+		if *file != "-" {
+			f, e := os.Open(*file)
+			if e != nil {
+				return e
+			}
+			defer f.Close()
+			reader = f
+		}
+		data, e := io.ReadAll(io.LimitReader(reader, (1<<20)+1))
+		if e != nil {
+			return e
+		}
+		if len(data) > 1<<20 {
+			return fmt.Errorf("CSV must be smaller than 1 MB")
+		}
+		values, err = community.ParseCSV(bytes.NewReader(data))
+		if err != nil {
+			return err
+		}
+	}
+	if !*credits && !*requestMode && (len(values) == 0 || len(values) > community.MaxURLs) {
+		return fmt.Errorf("provide 1–100 webpage URLs or -csv FILE")
+	}
+	for i, v := range values {
+		values[i], err = community.NormalizeURL(v)
+		if err != nil {
+			return fmt.Errorf("URL %d: %w", i+1, err)
+		}
+	}
+	jar, _ := cookiejar.New(nil)
+	client := &http.Client{Jar: jar, Timeout: 30 * time.Second, CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }}
+	call := func(path string, body any) ([]byte, error) {
+		b, _ := json.Marshal(body)
+		method := "POST"
+		if body == nil {
+			method = "GET"
+		}
+		req, err := http.NewRequestWithContext(ctx, method, strings.TrimRight(*server, "/")+"/api/"+path, bytes.NewReader(b))
+		if err != nil {
+			return nil, err
+		}
+		req.Header.Set("Content-Type", "application/json")
+		req.Header.Set("X-Cosift-Client", "community")
+		res, err := client.Do(req)
+		if err != nil {
+			return nil, err
+		}
+		defer res.Body.Close()
+		data, err := io.ReadAll(io.LimitReader(res.Body, 1<<20))
+		if err != nil {
+			return nil, err
+		}
+		if res.StatusCode < 200 || res.StatusCode >= 300 {
+			return nil, fmt.Errorf("community %s: HTTP %d: %s", path, res.StatusCode, strings.TrimSpace(string(data)))
+		}
+		return data, nil
+	}
+	if !*guest && *email != "" {
+		if _, err := call("login", map[string]string{"email": *email, "password": password}); err != nil {
+			return err
+		}
+		// Revoke this CLI session after use; browser sessions are separate.
+		defer func() { _, _ = call("logout", map[string]string{}) }()
+	}
+	var body any = map[string]any{"urls": values}
+	path := "submissions"
+	if *local || *credits {
+		if *guest || *email == "" {
+			return fmt.Errorf("local indexing and credits require email/password login")
+		}
+		if *local && *credits {
+			return fmt.Errorf("use -index-locally or -credits")
+		}
+	}
+	if *local {
+		artifacts, e := indexLocalContributions(ctx, cfg, values)
+		if e != nil {
+			return e
+		}
+		body = map[string]any{"artifacts": artifacts}
+		encoded, _ := json.Marshal(body)
+		if len(encoded) > 1<<20 {
+			return fmt.Errorf("local artifacts exceed 1 MB; submit fewer URLs")
+		}
+	}
+	if *requestMode {
+		if *local || *credits || len(values) > 0 || *file != "" {
+			return fmt.Errorf("request cannot be combined with contributions or credits")
+		}
+		if strings.TrimSpace(*query) == "" || (*mode != "search" && *mode != "answer" && *mode != "research") {
+			return fmt.Errorf("provide -query and a valid -mode")
+		}
+		path = *mode + "?q=" + url.QueryEscape(*query)
+		body = nil
+		client.Timeout = 4 * time.Minute
+	}
+	if *credits {
+		path = "credits"
+		body = nil
+	}
+	result, err := call(path, body)
+	if err != nil {
+		return err
+	}
+	_, err = os.Stdout.Write(result)
+	return err
+}