From 41bee926fe23d92cfd92162b9d4809b6153fa5e5 Mon Sep 17 00:00:00 2001 From: xsvm <2479667226@qq.com> Date: Sun, 6 Sep 2026 23:49:44 +0800 Subject: [PATCH] fix(read): sanitize unescaped C0 control chars before typed decode in getInto (#525) --- internal/cmd/read.go | 55 ++-------------------------------- pkg/civitai/read.go | 65 ++++++++++++++++++++++++++++++++++++++++ pkg/civitai/read_test.go | 37 +++++++++++++++++++++++ 3 files changed, 105 insertions(+), 52 deletions(-) diff --git a/internal/cmd/read.go b/internal/cmd/read.go index 50e50c8d..bbf09f31 100644 --- a/internal/cmd/read.go +++ b/internal/cmd/read.go @@ -108,59 +108,10 @@ func emitJSON(cmd *cobra.Command, raw []byte) error { return nil } -// escapeJSONStringControlChars walks raw JSON bytes tracking in-string state -// (respecting \\ and \") and replaces any raw C0 control byte (0x00–0x1F) that -// appears INSIDE a string literal with its valid JSON escape (\b \f \n \r \t or -// \u00xx). Control bytes outside strings (structural whitespace) and everything -// already escaped are left byte-for-byte unchanged, so valid input round-trips -// identically. This does not attempt to repair other kinds of malformed JSON; -// callers should verify the result with json.Valid before relying on it. +// escapeJSONStringControlChars delegates to civitai.EscapeJSONStringControlChars +// in pkg/civitai. func escapeJSONStringControlChars(raw []byte) []byte { - var out bytes.Buffer - out.Grow(len(raw)) - inString := false - escaped := false - for i := 0; i < len(raw); i++ { - c := raw[i] - switch { - case !inString: - out.WriteByte(c) - if c == '"' { - inString = true - } - case escaped: - // Previous byte was a backslash; this byte is the escape's payload - // (", \\, /, b, f, n, r, t, or the u of a \uXXXX). Emit verbatim. - out.WriteByte(c) - escaped = false - case c == '\\': - out.WriteByte(c) - escaped = true - case c == '"': - out.WriteByte(c) - inString = false - case c < 0x20: - // Raw control character inside a string literal — invalid JSON. - // Rewrite it as the shortest valid escape. - switch c { - case '\b': - out.WriteString(`\b`) - case '\f': - out.WriteString(`\f`) - case '\n': - out.WriteString(`\n`) - case '\r': - out.WriteString(`\r`) - case '\t': - out.WriteString(`\t`) - default: - fmt.Fprintf(&out, `\u%04x`, c) - } - default: - out.WriteByte(c) - } - } - return out.Bytes() + return civitai.EscapeJSONStringControlChars(raw) } // nonModelFileMarker returns a short human tag naming a version's PRIMARY file diff --git a/pkg/civitai/read.go b/pkg/civitai/read.go index 3c4f6169..f60c3c2a 100644 --- a/pkg/civitai/read.go +++ b/pkg/civitai/read.go @@ -1,6 +1,7 @@ package civitai import ( + "bytes" "context" "encoding/json" "errors" @@ -89,6 +90,10 @@ func (c *Client) getRaw(ctx context.Context, path string, q url.Values) (int, [] // getInto GETs path+q, and on a 2xx unmarshals the body into out (when non-nil) // and returns the raw body (for --json). A non-2xx returns a readError. +// Raw C0 control characters (0x00–0x1F) inside string literals — which violate +// strict RFC 8259 JSON syntax but are intermittently emitted by the Civitai API +// inside prompt/description strings — are sanitized via +// EscapeJSONStringControlChars before unmarshaling so typed decode succeeds. func (c *Client) getInto(ctx context.Context, path string, q url.Values, out any) ([]byte, error) { status, raw, err := c.getRaw(ctx, path, q) if err != nil { @@ -97,6 +102,11 @@ func (c *Client) getInto(ctx context.Context, path string, q url.Values, out any if status < 200 || status >= 300 { return nil, readError(status, raw) } + if !json.Valid(raw) { + if fixed := EscapeJSONStringControlChars(raw); json.Valid(fixed) { + raw = fixed + } + } if out != nil { if err := json.Unmarshal(raw, out); err != nil { return nil, fmt.Errorf("unexpected response from %s (status %d): %s", path, status, snippet(raw)) @@ -105,6 +115,61 @@ func (c *Client) getInto(ctx context.Context, path string, q url.Values, out any return raw, nil } +// EscapeJSONStringControlChars walks raw JSON bytes tracking in-string state +// (respecting \\ and \") and replaces any raw C0 control byte (0x00–0x1F) that +// appears INSIDE a string literal with its valid JSON escape (\b \f \n \r \t or +// \u00xx). Control bytes outside strings (structural whitespace) and everything +// already escaped are left byte-for-byte unchanged, so valid input round-trips +// identically. This does not attempt to repair other kinds of malformed JSON; +// callers should verify the result with json.Valid before relying on it. +func EscapeJSONStringControlChars(raw []byte) []byte { + var out bytes.Buffer + out.Grow(len(raw)) + inString := false + escaped := false + for i := 0; i < len(raw); i++ { + c := raw[i] + switch { + case !inString: + out.WriteByte(c) + if c == '"' { + inString = true + } + case escaped: + // Previous byte was a backslash; this byte is the escape's payload + // (", \\, /, b, f, n, r, t, or the u of a \uXXXX). Emit verbatim. + out.WriteByte(c) + escaped = false + case c == '\\': + out.WriteByte(c) + escaped = true + case c == '"': + out.WriteByte(c) + inString = false + case c < 0x20: + // Raw control character inside a string literal — invalid JSON. + // Rewrite it as the shortest valid escape. + switch c { + case '\b': + out.WriteString(`\b`) + case '\f': + out.WriteString(`\f`) + case '\n': + out.WriteString(`\n`) + case '\r': + out.WriteString(`\r`) + case '\t': + out.WriteString(`\t`) + default: + fmt.Fprintf(&out, `\u%04x`, c) + } + default: + out.WriteByte(c) + } + } + return out.Bytes() +} + // readError turns a non-2xx read response into a clear, actionable error, // surfacing the API's own error body ({"error": ...} or {"message": ...}) // rather than a Go struct dump. diff --git a/pkg/civitai/read_test.go b/pkg/civitai/read_test.go index 722a38c6..ffa145db 100644 --- a/pkg/civitai/read_test.go +++ b/pkg/civitai/read_test.go @@ -3,6 +3,7 @@ package civitai import ( "bytes" "context" + "encoding/json" "errors" "net/http" "net/http/httptest" @@ -310,3 +311,39 @@ func TestCursorStringEmpty(t *testing.T) { t.Errorf("empty metadata CursorString = %q, want empty", got) } } + +// TestGetIntoSanitizesRawControlChars verifies that an API response carrying +// an unescaped C0 control character inside a string literal (which violates +// strict RFC 8259 JSON syntax, but is intermittently emitted by Civitai) is +// sanitized before typed unmarshaling, so getInto succeeds instead of failing +// with an "unexpected response" error. +func TestGetIntoSanitizesRawControlChars(t *testing.T) { + // A payload where "name" carries a literal carriage return (0x0D). + raw := "{\"items\":[{\"id\":1,\"name\":\"Pony\rModel\",\"type\":\"Checkpoint\"}]}" + srv, _, _, _ := newTestServer(t, raw) + + c := New(srv.URL, "") + res, err := c.SearchModels(context.Background(), url.Values{}) + if err != nil { + t.Fatalf("SearchModels failed on raw CR payload: %v", err) + } + if len(res.Items) != 1 { + t.Fatalf("expected 1 item, got %d", len(res.Items)) + } + if res.Items[0].Name != "Pony\rModel" { + t.Errorf("item name = %q, want %q", res.Items[0].Name, "Pony\rModel") + } + if !json.Valid(res.Raw) { + t.Fatalf("res.Raw should be valid JSON, got: %q", res.Raw) + } +} + +// TestEscapeJSONStringControlCharsLeavesStructuralWhitespace ensures control +// bytes outside strings (newlines/tabs between tokens) are left untouched. +func TestEscapeJSONStringControlCharsLeavesStructuralWhitespace(t *testing.T) { + raw := []byte("{\n\t\"a\": 1\n}") + got := EscapeJSONStringControlChars(raw) + if !bytes.Equal(got, raw) { + t.Fatalf("structural whitespace altered:\n got %q\nwant %q", got, raw) + } +}