diff --git a/CLAUDE.md b/CLAUDE.md index bc513cd..df11a1c 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -95,6 +95,7 @@ go-sdk/ ├── instrumentation/ # tracing + metrics SPIs, no-op defaults, policies ├── config/ # layered override→env→default settings ├── serde/ # Marshaler/Unmarshaler seam + JSON default + Tristate +├── jsonl/ # NDJSON / JSON Lines stream decoder ├── sse/ # Server-Sent Events (WHATWG) parser ├── webhook/ # inbound signature verification (HMAC + timestamp) ├── formdata/ # multipart/form-data body builder diff --git a/README.md b/README.md index 0773f25..1653ba9 100644 --- a/README.md +++ b/README.md @@ -61,6 +61,7 @@ standard library. | [`config`](./config) | Layered override → environment → default settings resolver; non-failing typed getters. | | [`serde`](./serde) | Serialization seam (Marshaler/Unmarshaler) with a JSON default, plus Tristate for PATCH payloads. | | [`sse`](./sse) | Server-Sent Events (text/event-stream) WHATWG parser + reconnecting Stream (Last-Event-ID replay). | +| [`jsonl`](./jsonl) | JSON Lines / NDJSON streaming decoder (`iter.Seq2`). | | [`webhook`](./webhook) | Inbound webhook signature verification (constant-time HMAC + timestamp tolerance). | | [`formdata`](./formdata) | Multipart/form-data request body builder (replayable; file uploads). | | root [`dexpace`](.) | Umbrella `Client` wiring the default policy stack. | diff --git a/doc.go b/doc.go index c812461..1616f50 100644 --- a/doc.go +++ b/doc.go @@ -56,6 +56,9 @@ // JSON default) and Tristate for JSON PATCH payloads; httperr.ResponseError.DecodeInto // decodes an error body into a typed value. // +// The jsonl package decodes streaming JSON Lines (NDJSON) responses into typed +// values. +// // The sse package parses Server-Sent Events (text/event-stream) into a // range-over-func iterator of events, with a reconnecting Stream that replays the // Last-Event-ID. diff --git a/docs/superpowers/plans/2026-06-16-jsonl.md b/docs/superpowers/plans/2026-06-16-jsonl.md new file mode 100644 index 0000000..f334ff2 --- /dev/null +++ b/docs/superpowers/plans/2026-06-16-jsonl.md @@ -0,0 +1,289 @@ +# JSONL / NDJSON Decoder Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Add a `jsonl` package with `Decode[T](r io.Reader) iter.Seq2[T, error]` that streams successive JSON values from a reader (NDJSON / JSON Lines) as typed values. + +**Architecture:** A `json.Decoder` reads successive values from `r`; the iterator yields each decoded `T`, ends cleanly at `io.EOF`, and surfaces any other decode error once before stopping. + +**Tech Stack:** Go 1.26+, standard library only (`encoding/json`, `errors`, `io`, `iter`). Zero third-party dependencies. + +**Conventions every task must follow:** +- MIT license header on every `.go` file before the `package` clause. +- Import groups: stdlib only here. +- Tests use `t.Parallel()`; table-driven where natural; stdlib-only. +- Tools: Go 1.26.3; `gofumpt`/`golangci-lint` NOT installed — use `gofmt`, `go vet`, `go test -race`. +- Run commands from the repo root `/Users/omar/dexpace/go-sdk`. + +--- + +## File Structure + +| Path | Responsibility | +|---|---| +| `jsonl/doc.go` (new) | package comment | +| `jsonl/jsonl.go` (new) | `Decode` | +| `jsonl/jsonl_test.go` (new) | NDJSON / scalar / empty / error / break tests | +| `doc.go`, `README.md`, `CLAUDE.md` (modify) | document; add the package | + +--- + +## Task 1: the `jsonl` package + +**Files:** +- Create: `jsonl/doc.go`, `jsonl/jsonl.go`, `jsonl/jsonl_test.go` + +- [ ] **Step 1: Write the failing tests** + +```go +// jsonl/jsonl_test.go +// Copyright (c) 2026 dexpace and Omar Aljarrah. +// Licensed under the MIT License. See LICENSE in the repository root for details. + +package jsonl_test + +import ( + "strings" + "testing" + + "github.com/dexpace/go-sdk/jsonl" +) + +type rec struct { + N int `json:"n"` +} + +func collectRecs(t *testing.T, input string) ([]rec, error) { + t.Helper() + var got []rec + var gotErr error + for v, err := range jsonl.Decode[rec](strings.NewReader(input)) { + if err != nil { + gotErr = err + break + } + got = append(got, v) + } + return got, gotErr +} + +func TestDecodeNDJSON(t *testing.T) { + t.Parallel() + + got, err := collectRecs(t, "{\"n\":1}\n{\"n\":2}\n{\"n\":3}\n") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(got) != 3 || got[0].N != 1 || got[1].N != 2 || got[2].N != 3 { + t.Fatalf("got %v, want n=1,2,3", got) + } +} + +func TestDecodeSingleNoTrailingNewline(t *testing.T) { + t.Parallel() + + got, err := collectRecs(t, "{\"n\":5}") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(got) != 1 || got[0].N != 5 { + t.Fatalf("got %v, want one value n=5", got) + } +} + +func TestDecodeScalars(t *testing.T) { + t.Parallel() + + var got []int + for v, err := range jsonl.Decode[int](strings.NewReader("1 2 3")) { + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + got = append(got, v) + } + if len(got) != 3 || got[0] != 1 || got[1] != 2 || got[2] != 3 { + t.Fatalf("got %v, want [1 2 3]", got) + } +} + +func TestDecodeEmpty(t *testing.T) { + t.Parallel() + + got, err := collectRecs(t, "") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(got) != 0 { + t.Fatalf("got %v, want no values", got) + } +} + +func TestDecodeMalformedMidStream(t *testing.T) { + t.Parallel() + + got, err := collectRecs(t, "{\"n\":1}\n{bad}\n") + if err == nil { + t.Fatal("expected a decode error for the malformed value") + } + if len(got) != 1 || got[0].N != 1 { + t.Fatalf("got %v, want the first value before the error", got) + } +} + +func TestDecodeTruncated(t *testing.T) { + t.Parallel() + + got, err := collectRecs(t, "{\"n\":1}\n{\"n\":") + if err == nil { + t.Fatal("expected an error for the truncated final value") + } + if len(got) != 1 || got[0].N != 1 { + t.Fatalf("got %v, want the first value before the truncation", got) + } +} + +func TestDecodeEarlyBreak(t *testing.T) { + t.Parallel() + + count := 0 + for range jsonl.Decode[rec](strings.NewReader("{\"n\":1}\n{\"n\":2}\n{\"n\":3}\n")) { + count++ + break + } + if count != 1 { + t.Fatalf("consumed %d values, want 1 after break", count) + } +} +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `go test ./jsonl/ -v` +Expected: FAIL — `jsonl.Decode` undefined. + +- [ ] **Step 3: Create `jsonl/doc.go`** + +```go +// jsonl/doc.go +// Copyright (c) 2026 dexpace and Omar Aljarrah. +// Licensed under the MIT License. See LICENSE in the repository root for details. + +// Package jsonl decodes a stream of JSON values (JSON Lines / NDJSON) into typed +// values via a range-over-func iterator. Values may be separated by any JSON +// whitespace, so newline-delimited streams and concatenated values both work. +package jsonl +``` + +- [ ] **Step 4: Create `jsonl/jsonl.go`** + +```go +// jsonl/jsonl.go +// Copyright (c) 2026 dexpace and Omar Aljarrah. +// Licensed under the MIT License. See LICENSE in the repository root for details. + +package jsonl + +import ( + "encoding/json" + "errors" + "io" + "iter" +) + +// Decode reads a stream of JSON values from r and yields each decoded into a T. +// Values may be separated by any JSON whitespace (newlines for NDJSON / JSON +// Lines, or none). Iteration ends at end of stream; a decode error is delivered +// as the second iteration value, after which iteration stops. The iterator is +// single-pass. +func Decode[T any](r io.Reader) iter.Seq2[T, error] { + return func(yield func(T, error) bool) { + dec := json.NewDecoder(r) + for { + var v T + err := dec.Decode(&v) + if errors.Is(err, io.EOF) { + return + } + if err != nil { + var zero T + yield(zero, err) + return + } + if !yield(v, nil) { + return + } + } + } +} +``` + +- [ ] **Step 5: Run tests to verify they pass** + +Run: `go test ./jsonl/ -v` +Expected: PASS — NDJSON, single-value, scalars, empty, malformed, truncated, early-break. + +- [ ] **Step 6: Commit** + +```bash +git add jsonl/ +git commit -m "feat(jsonl): add NDJSON/JSON-Lines stream decoder" +``` + +--- + +## Task 2: docs and full gate + +**Files:** +- Modify: `doc.go`, `README.md`, `CLAUDE.md` + +- [ ] **Step 1: Mention jsonl in `doc.go`** + +Read `doc.go`. Within the `package dexpace` doc comment (single contiguous `//` +block; no second package clause / no duplicate header), add: + +```go +// The jsonl package decodes streaming JSON Lines (NDJSON) responses into typed +// values. +``` + +- [ ] **Step 2: Add `jsonl` to `README.md`** + +Read `README.md`. Add a `jsonl` row to the architecture/package table (matching +the column/link style): "JSON Lines / NDJSON streaming decoder (iter.Seq2)." + +- [ ] **Step 3: Add `jsonl/` to `CLAUDE.md` Repository Layout** + +Read `CLAUDE.md`. Add a `jsonl/` line near the other value-layer/streaming +packages: `jsonl/ # NDJSON / JSON Lines stream decoder`. + +- [ ] **Step 4: Run the full gate** + +Run: +```bash +gofmt -l . +go vet ./... +go test -race ./... +``` +Expected: `gofmt -l .` prints nothing; `go vet` clean; every package passes under +the race detector (`jsonl` now has tests). + +- [ ] **Step 5: Commit** + +```bash +git add doc.go README.md CLAUDE.md +git commit -m "docs: document the jsonl package" +``` + +--- + +## Self-Review notes (for the implementer) + +- **Spec coverage:** `Decode[T]` with EOF-clean / error-surface / break semantics + (Task 1); docs (Task 2). +- **Type consistency:** `jsonl.Decode[T](io.Reader) iter.Seq2[T, error]` used + identically across tasks/tests. +- **Clean EOF vs error:** `errors.Is(err, io.EOF)` ends cleanly; + `io.ErrUnexpectedEOF` (truncated value) and JSON syntax errors fall to the error + branch — verified by `TestDecodeTruncated` / `TestDecodeMalformedMidStream`. +- **Bounded:** `json.Decoder` reads incrementally; no whole-stream buffering added. +- **`make check`** green before opening the PR. diff --git a/docs/superpowers/specs/2026-06-16-jsonl-design.md b/docs/superpowers/specs/2026-06-16-jsonl-design.md new file mode 100644 index 0000000..395a92d --- /dev/null +++ b/docs/superpowers/specs/2026-06-16-jsonl-design.md @@ -0,0 +1,85 @@ +# JSONL / NDJSON stream decoder — design + +**Date:** 2026-06-16 +**Status:** Approved (standing delegation); ready for implementation planning +**Subsystem:** deferred-feature #3 (JSONL/chunked stream helpers from the HTTP-value-types roadmap item) + +## Context + +Streaming JSON APIs return newline-delimited JSON (NDJSON / JSON Lines): one JSON +value per chunk. Consuming them by hand (decoder loop, EOF handling) is repetitive. +This package decodes such a stream into typed values via a range-over-func +iterator, matching `pagination` and `sse`. + +## Decisions + +1. **Generic stream decoder.** `Decode[T](r io.Reader) iter.Seq2[T, error]` yields + each successive JSON value decoded into a `T`. +2. **Lean on `encoding/json`.** A `json.Decoder` reads successive values from the + stream, tolerating any whitespace (including newlines) between them — so NDJSON, + pretty-printed concatenation, and a single value all work. No custom line + splitting. +3. **Bounded by the decoder.** `json.Decoder` reads incrementally; the package + adds no unbounded buffering of the whole stream. +4. **Own small package `jsonl`.** Named for what it provides; discoverable at the + call site (`jsonl.Decode`). + +## Architecture + +### `jsonl` package (stdlib-only) + +```go +// Decode reads a stream of JSON values from r and yields each decoded into a T. +// Values may be separated by any JSON whitespace (newlines for NDJSON / JSON +// Lines, or none). Iteration ends at end of stream; a decode error is delivered +// as the second iteration value, after which iteration stops. The iterator is +// single-pass. +func Decode[T any](r io.Reader) iter.Seq2[T, error] +``` + +Implementation: a `json.Decoder` over `r`; loop `dec.Decode(&v)`; `io.EOF` ends +iteration cleanly; any other error is yielded once (with the zero `T`) and stops; +each successfully decoded value is yielded; a consumer break stops decoding. + +## Edge cases + +- An empty stream → no values, no error. +- Trailing whitespace/newline after the last value → clean EOF, no spurious value. +- A malformed value mid-stream → the decode error is yielded once, then iteration + stops (no attempt to resynchronize). +- A partial value at EOF (truncated stream) → `json.Decoder` returns an + `io.ErrUnexpectedEOF`-class error, yielded as the iteration error. +- Early `break` from the iterator stops decoding (range-over-func semantics). +- The element type may be any JSON-decodable type (struct, map, slice, scalar). + +## Package layout + +| Path | Change | +|---|---| +| `jsonl/doc.go` (new) | package comment | +| `jsonl/jsonl.go` (new) | `Decode` | +| `jsonl/jsonl_test.go` (new) | NDJSON, single value, empty, error, break tests | +| `doc.go`, `README.md`, `CLAUDE.md` | document; add the package | + +## Testing + +- NDJSON: three `{"n":...}` objects on separate lines → three values in order. +- Whitespace tolerance: values separated by extra spaces/newlines decode the same; + a single value (no trailing newline) decodes. +- Scalars: a stream of bare numbers (`1 2 3`) decodes into `int`s. +- Empty stream → zero values, no error. +- A malformed value mid-stream → the first values decode, then the error is + yielded and iteration stops. +- Truncated final value → an error is yielded. +- Early `break` stops decoding (a counting reader or asserting only N consumed). +- Table-driven where natural, parallel; stdlib-only; `gofmt`/`go vet`/`go test + -race` clean. + +## Out of scope (deferred) + +- Chunked-transfer framing helpers (net/http already de-chunks the body; the + decoded stream is what `Decode` consumes). +- An `Encode` (NDJSON writer) — callers can `json.Marshal` + write a newline; add + if a real need appears. +- Resynchronizing after a malformed value (skip-and-continue) — stops on first + error, matching the other iterators. diff --git a/jsonl/doc.go b/jsonl/doc.go new file mode 100644 index 0000000..2514bf4 --- /dev/null +++ b/jsonl/doc.go @@ -0,0 +1,7 @@ +// Copyright (c) 2026 dexpace and Omar Aljarrah. +// Licensed under the MIT License. See LICENSE in the repository root for details. + +// Package jsonl decodes a stream of JSON values (JSON Lines / NDJSON) into typed +// values via a range-over-func iterator. Values may be separated by any JSON +// whitespace, so newline-delimited streams and concatenated values both work. +package jsonl diff --git a/jsonl/jsonl.go b/jsonl/jsonl.go new file mode 100644 index 0000000..cddef96 --- /dev/null +++ b/jsonl/jsonl.go @@ -0,0 +1,37 @@ +// Copyright (c) 2026 dexpace and Omar Aljarrah. +// Licensed under the MIT License. See LICENSE in the repository root for details. + +package jsonl + +import ( + "encoding/json" + "errors" + "io" + "iter" +) + +// Decode reads a stream of JSON values from r and yields each decoded into a T. +// Values may be separated by any JSON whitespace (newlines for NDJSON / JSON +// Lines, or none). Iteration ends at end of stream; a decode error is delivered +// as the second iteration value, after which iteration stops. The iterator is +// single-pass. +func Decode[T any](r io.Reader) iter.Seq2[T, error] { + return func(yield func(T, error) bool) { + dec := json.NewDecoder(r) + for { + var v T + err := dec.Decode(&v) + if errors.Is(err, io.EOF) { + return + } + if err != nil { + var zero T + yield(zero, err) + return + } + if !yield(v, nil) { + return + } + } + } +} diff --git a/jsonl/jsonl_test.go b/jsonl/jsonl_test.go new file mode 100644 index 0000000..62cbb82 --- /dev/null +++ b/jsonl/jsonl_test.go @@ -0,0 +1,117 @@ +// Copyright (c) 2026 dexpace and Omar Aljarrah. +// Licensed under the MIT License. See LICENSE in the repository root for details. + +package jsonl_test + +import ( + "strings" + "testing" + + "github.com/dexpace/go-sdk/jsonl" +) + +type rec struct { + N int `json:"n"` +} + +func collectRecs(t *testing.T, input string) ([]rec, error) { + t.Helper() + var got []rec + var gotErr error + for v, err := range jsonl.Decode[rec](strings.NewReader(input)) { + if err != nil { + gotErr = err + break + } + got = append(got, v) + } + return got, gotErr +} + +func TestDecodeNDJSON(t *testing.T) { + t.Parallel() + + got, err := collectRecs(t, "{\"n\":1}\n{\"n\":2}\n{\"n\":3}\n") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(got) != 3 || got[0].N != 1 || got[1].N != 2 || got[2].N != 3 { + t.Fatalf("got %v, want n=1,2,3", got) + } +} + +func TestDecodeSingleNoTrailingNewline(t *testing.T) { + t.Parallel() + + got, err := collectRecs(t, "{\"n\":5}") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(got) != 1 || got[0].N != 5 { + t.Fatalf("got %v, want one value n=5", got) + } +} + +func TestDecodeScalars(t *testing.T) { + t.Parallel() + + var got []int + for v, err := range jsonl.Decode[int](strings.NewReader("1 2 3")) { + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + got = append(got, v) + } + if len(got) != 3 || got[0] != 1 || got[1] != 2 || got[2] != 3 { + t.Fatalf("got %v, want [1 2 3]", got) + } +} + +func TestDecodeEmpty(t *testing.T) { + t.Parallel() + + got, err := collectRecs(t, "") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(got) != 0 { + t.Fatalf("got %v, want no values", got) + } +} + +func TestDecodeMalformedMidStream(t *testing.T) { + t.Parallel() + + got, err := collectRecs(t, "{\"n\":1}\n{bad}\n") + if err == nil { + t.Fatal("expected a decode error for the malformed value") + } + if len(got) != 1 || got[0].N != 1 { + t.Fatalf("got %v, want the first value before the error", got) + } +} + +func TestDecodeTruncated(t *testing.T) { + t.Parallel() + + got, err := collectRecs(t, "{\"n\":1}\n{\"n\":") + if err == nil { + t.Fatal("expected an error for the truncated final value") + } + if len(got) != 1 || got[0].N != 1 { + t.Fatalf("got %v, want the first value before the truncation", got) + } +} + +func TestDecodeEarlyBreak(t *testing.T) { + t.Parallel() + + count := 0 + for range jsonl.Decode[rec](strings.NewReader("{\"n\":1}\n{\"n\":2}\n{\"n\":3}\n")) { + count++ + break + } + if count != 1 { + t.Fatalf("consumed %d values, want 1 after break", count) + } +}