docs/imporve documentaions

README.md

@@ -32,9 +32,6 @@ Install via [npm](https://www.npmjs.com/package/jsontext):
 npm install jsontext
 ```
 
-> [!NOTE]
-> It may require Node.js 18 or later.
-
 ### Deno
 
 Install via [JSR](https://jsr.io/@lcweden/jsontext):
@@ -92,8 +89,8 @@ for await (const value of titles) {
 
 Represents JSON at two granularities:
 
-- **Tokens**: The smallest lexical unit (a scalar like `"Alice"`, `true`, `123`, or a delimiter like
-  `{`, `}`, `[`, `]`).
+- **Tokens**: The smallest lexical unit (a scalar like `"Alice"`, `true`, `123`, or a structural
+  symbol like `{`, `}`, `[`, `]`).
 
 - **Values**: A complete unit — a scalar, or an entire `object` or `array` including everything
   nested inside.
@@ -172,7 +169,7 @@ decoder.checkEOF();
 fetch more bytes, not an error. `end()` tells the decoder no more input is coming; `checkEOF()` then
 asserts that what arrived was a complete, well-formed document.
 
-### Composing with streams
+### Composing
 
 The core `JSONTextDecoder` and `JSONTextEncoder` are manual state machines. For common use cases,
 use `TransformStream` wrappers that natively compose with fetch, files, and Web Streams.
@@ -181,7 +178,7 @@ use `TransformStream` wrappers that natively compose with fetch, files, and Web
 import { JSONTextLineStream, JSONTextSelectorStream } from "jsontext";
 
 // Filter a JSON Lines feed: keep only active users, write them back out as JSONL.
-// JSONTextLineStream emits one Value per line, preserving the original bytes.
+// JSONTextLineStream emits one Value per top-level JSON value — ideal for JSONL and concatenated-JSON.
 const encoder = new TextEncoder();
 
 await response.body
@@ -222,60 +219,46 @@ try {
 }
 ```
 
-## Example Pipelines
+## Examples
+
+Below are some simple examples demonstrating how to use `jsontext` for common JSON processing tasks.
+For more examples, see the [documentation](/docs/).
 
 ### Replace `null` with an empty string
 
-Swap every `null` token for an empty string as the JSON flows through — no parsing the whole
-document, no intermediate object.
+In this example, we read a JSON stream from an API endpoint, replace all `null` values with empty
+strings, and write the modified JSON back out as a stream without ever materializing the whole
+document in memory.
 
 ```javascript
 import { JSONTextDecoderStream, JSONTextEncoderStream, KIND, Token } from "jsontext";
 
-stream
-  .pipeThrough(new JSONTextDecoderStream()) // decode bytes into tokens
-  .pipeThrough(
-    new TransformStream({
-      transform(token, controller) {
-        if (token.kind === KIND.NULL) { // Detect a `null` token
-          controller.enqueue(Token.fromString("")); // Emit an empty string token instead
-        } else {
-          controller.enqueue(token);
-        }
-      },
-    }),
-  )
-  .pipeThrough(new JSONTextEncoderStream()); // encode tokens back into bytes
-```
-
-### Extract and Restructure Data
+const response = await fetch("your.api/endpoint");
 
-Extract specific nested elements using JSONPath, and wrap them into a brand new JSON array structure
-directly in the stream pipeline.
-
-```javascript
-import { JSONTextEncoderStream, JSONTextSelectorStream, Token } from "jsontext";
+if (!response.ok || !response.body) {
+  throw new Error("Failed to fetch data");
+}
 
-stream
-  .pipeThrough(new JSONTextSelectorStream("$.todos[*].todo")) // extract all `todo` values from the `todos` array
-  .pipeThrough(
-    new TransformStream({
-      start(controller) {
-        controller.enqueue(Token.ARRAY_BEGIN); // emit a `[` to start the output array
-      },
-      transform(value, controller) {
-        for (const token of value.tokens()) {
-          controller.enqueue(token);
-        }
-      },
-      flush(controller) {
-        controller.enqueue(Token.ARRAY_END); // emit a `]` to end the output array
-      },
-    }),
-  )
-  .pipeThrough(new JSONTextEncoderStream()); // encode back to bytes for output
+const decoder = new JSONTextDecoderStream();
+const encoder = new JSONTextEncoderStream();
+const replacer = new TransformStream({
+  transform(token, controller) {
+    if (token.kind === KIND.NULL) { // Detect a `null` token
+      controller.enqueue(Token.fromString("")); // Emit an empty string token instead
+    } else {
+      controller.enqueue(token);
+    }
+  },
+});
+
+const stream = response.body.pipeThrough(decoder).pipeThrough(replacer).pipeThrough(encoder);
+const blob = await new Response(stream).blob();
 ```
 
+> [!TIP]
+> `JSONTextDecoderStream` supports Token-level processing only. If you need to replace values that
+> may be nested inside objects or arrays, you will need to use `JSONTextDecoder` directly.
+
 ## License
 
 This project is licensed under the [MIT](LICENSE) License.

src/api/decoder.ts

@@ -13,6 +13,8 @@ type JSONTextDecoderOptions = DecoderOptions;
  * Feed byte chunks via {@link push} then consume tokens with
  * {@link readToken} / {@link readValue} / {@link skipValue}.
  * Call {@link end} when the stream is exhausted to flush any buffered state.
+ *
+ * @public
  */
 class JSONTextDecoder {
   #decoder: Decoder;
@@ -28,15 +30,17 @@ class JSONTextDecoder {
   /**
    * Asserts that the input has been fully consumed.
    *
-   * @throws {SyntacticError} If there are unread bytes remaining.
+   * @throws {SyntaxError} If the decoder is still inside a nested structure,
+   *   or if non-whitespace characters remain after the value.
    */
   checkEOF(): void {
     this.#decoder.checkEOF();
   }
 
   /**
-   * The current nesting depth — `0` at top level, incremented inside each
-   * object or array.
+   * Returns the current nesting depth of the structural state.
+   *
+   * @returns The current nesting depth — `1` at the top level, incremented by each open object or array.
    */
   depth(): number {
     return this.#decoder.depth();
@@ -45,18 +49,18 @@ class JSONTextDecoder {
   /**
    * Signals that no more input will be pushed.
    *
-   * Validates that any incomplete value is properly terminated.
-   *
-   * @throws {SyntacticError} If the input ends in the middle of a value.
+   * After calling this, number tokens no longer require a trailing byte to
+   * confirm their end.
    */
   end(): void {
     this.#decoder.end();
   }
 
   /**
-   * The byte offset of the next unread byte within the total input seen so far.
+   * The byte offset of the end of the last consumed token within the total
+   * input seen so far.
    *
-   * @returns The byte offset of the next unread byte, or the total length of all
+   * @returns The global byte offset from the start of the stream.
    */
   inputOffset(): number {
     return this.#decoder.inputOffset();
@@ -76,6 +80,7 @@ class JSONTextDecoder {
    * or `undefined` if no complete token is available yet.
    *
    * @returns The {@link Kind} of the next token, or `undefined` if no complete token is available yet.
+   * @throws {SyntacticError} If an invalid character or unexpected delimiter is encountered.
    */
   peekKind(): Kind | undefined {
     return this.#decoder.peekKind();
@@ -125,11 +130,11 @@ class JSONTextDecoder {
    * Returns a JSON Pointer string describing a position in the current
    * nesting context.
    *
-   * | `where` | Meaning |
-   * |---------|-------------------------------------------------------------------|
+   * | `where` | Meaning                                                  |
+   * |---------|----------------------------------------------------------|
    * | `1`     | The position of the **next** value to be read (default). |
-   * | `0`     | The position of the **current** container. |
-   * | `-1`    | The position of the **previously** read value. |
+   * | `0`     | The position of the **current** container.               |
+   * | `-1`    | The position of the **previously** read value.           |
    *
    * @param where - Which position to return. Defaults to `1`.
    * @returns A JSON Pointer string, e.g. `"/foo/0"`.

src/api/encoder.ts

@@ -12,6 +12,8 @@ type JSONTextEncoderOptions = EncoderOptions;
  * Write tokens or values via {@link writeToken} / {@link writeValue}, then
  * retrieve the accumulated output with {@link bytes}. Call {@link reset} to
  * start a new document without creating a new instance.
+ *
+ * @public
  */
 class JSONTextEncoder {
   #encoder: Encoder;
@@ -24,10 +26,10 @@ class JSONTextEncoder {
   }
 
   /**
-   * The current nesting depth — `0` at top level, incremented inside each
-   * object or array.
+   * The current nesting depth — `1` at the top level, incremented by each
+   * open object or array.
    *
-   * @returns The current nesting depth.
+   * @returns The current nesting depth — `1` at the top level, incremented by each open object or array.
    */
   depth(): number {
     return this.#encoder.depth();
@@ -72,6 +74,8 @@ class JSONTextEncoder {
    * Drains and returns the bytes accumulated in the output buffer since the
    * last call. The internal buffer is cleared; structural state (nesting,
    * delimiters) is preserved so subsequent writes continue the same document.
+   *
+   * @returns A copy of the bytes written since the last `takeBytes` call.
    */
   takeBytes(): Uint8Array {
     return this.#encoder.takeBytes();

src/common/constants.ts

@@ -1,3 +1,4 @@
+/** ASCII byte values used for JSON parsing and encoding. */
 const ASCII = {
   TAB: 0x09,
   LINE_FEED: 0x0A,
@@ -51,6 +52,7 @@ const ASCII = {
   DELETE: 0x7F,
 } as const;
 
+/** Unicode escape sequences for safe HTML and JavaScript embedding. */
 const UNICODE = {
   OPEN_ANGLED_BRACKET: "\\u003c",
   CLOSE_ANGLED_BRACKET: "\\u003e",
@@ -59,6 +61,7 @@ const UNICODE = {
   PARAGRAPH_SEPARATOR: "\\u2029",
 } as const;
 
+/** String discriminants identifying the structural role of a JSON token. */
 const KIND = {
   NULL: "null",
   FALSE: "false",
@@ -71,13 +74,16 @@ const KIND = {
   ARRAY_END: "]",
 } as const;
 
+/** Maximum JSON nesting depth supported by the decoder and encoder. */
 const MAX_NESTING_DEPTH = 10_000;
 
+/** Default option values for decoding. */
 const DEFAULT_DECODER_OPTIONS = {
   allowDuplicateNames: false,
   allowInvalidUTF8: false,
 } as const;
 
+/** Default option values for encoding. */
 const DEFAULT_ENCODER_OPTIONS = {
   escapeForHTML: false,
   escapeForJS: false,
@@ -89,16 +95,19 @@ const DEFAULT_ENCODER_OPTIONS = {
   indentPrefix: "",
 } as const;
 
+/** JSON Path identifier types. */
 const IDENTIFIER = {
   ROOT: 0,
   CURRENT: 1,
 } as const;
 
+/** JSON Path segment kinds. */
 const SEGMENT = {
   CHILD: 0,
   DESCENDANT: 1,
 } as const;
 
+/** JSON Path selector kinds. */
 const SELECTOR = {
   NAME: 0,
   WILDCARD: 1,

src/libs/stream-decoder.ts

@@ -15,54 +15,50 @@ type JSONTextDecoderStreamOptions = DecoderOptions & {
  * Writable side accepts raw JSON bytes (possibly split across multiple chunks).
  * Readable side emits one {@link Token} per JSON token in document order.
  *
+ * @public
  * @example
+ * ```javascript
  * const response = await fetch(url);
  * const tokens = response.body
  *   .pipeThrough(new JSONTextDecoderStream());
+ * ```
  */
 class JSONTextDecoderStream extends TransformStream<Uint8Array, Token> {
+  #decoder: Decoder;
+
   /**
    * @param options - Decoder and queuing strategy options.
    */
   constructor(options: JSONTextDecoderStreamOptions = {}) {
     const { writableStrategy, readableStrategy, ...rest } = options;
     const decoderOptions = { ...DEFAULT_DECODER_OPTIONS, ...rest };
-    const decoder = new Decoder(new Uint8Array(), decoderOptions);
 
     super(
       {
-        transform(chunk, controller) {
-          try {
-            decoder.push(chunk);
-
-            let token;
-
-            while ((token = decoder.readToken()) !== undefined) {
-              controller.enqueue(token);
-            }
-          } catch (error) {
-            controller.error(error);
-          }
+        transform: (chunk, controller) => {
+          this.#decoder.push(chunk);
+          this.#drain(controller);
         },
-        flush(controller) {
-          try {
-            decoder.end();
-
-            let token;
-
-            while ((token = decoder.readToken()) !== undefined) {
-              controller.enqueue(token);
-            }
-
-            decoder.checkEOF();
-          } catch (error) {
-            controller.error(error);
-          }
+        flush: (controller) => {
+          this.#decoder.end();
+          this.#drain(controller);
+          this.#decoder.checkEOF();
         },
       },
       writableStrategy,
       readableStrategy,
     );
+
+    this.#decoder = new Decoder(new Uint8Array(), decoderOptions);
+  }
+
+  /**
+   * Drains all available tokens from the decoder into the readable side.
+   */
+  #drain(controller: TransformStreamDefaultController<Token>): void {
+    for (let token; (token = this.#decoder.readToken()) !== undefined;) {
+      controller.enqueue(token);
+    }
   }
 }