Skip to content

Commit 35b345e

Browse files
committed
fixup(errs): address review on the HTTP classifier
- Stop declining context.DeadlineExceeded. Only context.Canceled is left to the generic classifier now. A deadline that elapses mid-request means the remote end did not answer in time, so it belongs to the dependency; declining it also stranded the node, because generic matches only Canceled and nothing else claimed it unless platform/errs/mysql happened to be wired. A processor test with just generic + http pins that the classifier stands on its own. - Bound StatusError.Error to 1 KiB of body. The rendered string reaches the queue's dead-letter record through Reject(ctx, err.Error()) and last_error is finite, so a large error page could fail that write and leave the message stuck instead of dead-lettered. Body is still kept whole. - Convert the GitHub Actions client's two status paths as well. Stovepipe runs both build runners, so leaving one untyped meant a 502 was retried or dead-lettered depending only on which runner the queue used. - Assert with errors.As that both clients return *StatusError and carry the code. The old tests only checked that an error came back, so they passed before the typed error existed and would keep passing if it were reverted. - Document that a retried create can duplicate a build, on both clients' create paths. Note that codes below 100, including 0, take the non-retryable fall-through deliberately, and cover 0 in the table.
1 parent 3171362 commit 35b345e

8 files changed

Lines changed: 129 additions & 36 deletions

File tree

platform/errs/http/http.go

Lines changed: 15 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -79,38 +79,26 @@ func (classifier) Classify(err error) errs.Verdict {
7979
}
8080

8181
if ue, ok := err.(*url.Error); ok {
82-
// A cancelled or expired context means we gave up, not that the remote
83-
// end misbehaved: the deadline belongs to this process. Report Unknown
84-
// so the walk continues into the cause, where the generic classifier
85-
// picks up context.Canceled as plain retryable infra. That keeps
86-
// shutdown noise out of the dependency-error metrics for this backend.
87-
// Reading the Err field is not a chain walk, and declining here leaves
88-
// the framework's walk free to reach that node.
89-
//
90-
// This covers the shape http.Client.Do produces for context failures. A
91-
// context error buried under further transport wrapping falls through to
92-
// the retryable dependency verdict below, which retries the same way and
93-
// differs only in the dependency tag.
94-
if ue.Err == context.Canceled || ue.Err == context.DeadlineExceeded {
82+
// A cancelled context is ours, not theirs — process shutdown, or a parent
83+
// operation that went away — so decline it and let the generic classifier
84+
// claim context.Canceled as plain retryable infra, keeping shutdowns out
85+
// of this backend's dependency metrics. An expired deadline is theirs:
86+
// the remote end did not answer in time, so it takes the verdict below.
87+
// Declining that one would strand it, since generic matches only Canceled.
88+
if ue.Err == context.Canceled {
9589
return errs.Unknown
9690
}
97-
// Anything else at this layer is a failure to complete an exchange with
98-
// the remote end — reset connection, refused dial, DNS, TLS, client
99-
// timeout. None of those say the request was invalid, so another attempt
100-
// is worth making.
91+
// Everything else at this layer is a failed exchange with the remote end,
92+
// and none of those shapes says the request was invalid.
10193
return errs.InfraDependencyRetryable
10294
}
10395

10496
return errs.Unknown
10597
}
10698

107-
// classifyStatusCode maps an HTTP status code to a Verdict.
108-
//
109-
// The split is whether the code describes the state of the server or a verdict
110-
// on the request. Server-state codes change on their own, so the same request
111-
// can succeed later. Request-verdict codes will not: replaying the request
112-
// reproduces the answer, and the message should dead-letter for a human or a
113-
// reconciler instead of retrying until it runs out of attempts.
99+
// classifyStatusCode maps an HTTP status code to a Verdict. The split is whether
100+
// the code describes the state of the server, which can change on its own, or a
101+
// verdict on the request, which replaying only reproduces.
114102
func classifyStatusCode(code int) errs.Verdict {
115103
switch code {
116104
case nethttp.StatusRequestTimeout, // 408 — the server stopped waiting; sending it again is reasonable.
@@ -130,7 +118,8 @@ func classifyStatusCode(code int) errs.Verdict {
130118
}
131119

132120
// 4xx other than the two above, 3xx the client was not configured to follow,
133-
// and anything else a caller chose to reject: a verdict on this request. Not
134-
// retryable.
121+
// and anything else a caller chose to reject — including a code that was
122+
// never a response, such as 0: a verdict on the request, or on a malformed
123+
// call. Neither changes on a second attempt.
135124
return errs.InfraDependency
136125
}

platform/errs/http/http_test.go

Lines changed: 37 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,9 @@ func TestClassifier_StatusCodes(t *testing.T) {
5454
{"not found", nethttp.StatusNotFound, errs.InfraDependency},
5555
{"unprocessable entity", nethttp.StatusUnprocessableEntity, errs.InfraDependency},
5656
{"unfollowed redirect", nethttp.StatusFound, errs.InfraDependency},
57+
58+
// Never a response: a caller built this from something else.
59+
{"zero code", 0, errs.InfraDependency},
5760
}
5861

5962
for _, tt := range tests {
@@ -87,9 +90,10 @@ func TestClassifier_TransportFailures(t *testing.T) {
8790
want: errs.Unknown,
8891
},
8992
{
93+
// Theirs, not ours: the remote end did not answer in time.
9094
name: "context deadline exceeded",
9195
err: &url.Error{Op: "Get", URL: "http://api.example", Err: context.DeadlineExceeded},
92-
want: errs.Unknown,
96+
want: errs.InfraDependencyRetryable,
9397
},
9498
}
9599

@@ -155,6 +159,13 @@ func TestClassifier_AppliedViaProcessor(t *testing.T) {
155159
assert.False(t, errs.IsDependencyError(out))
156160
})
157161

162+
t.Run("expired deadline is retryable without mysqlerrs claiming it", func(t *testing.T) {
163+
err := fmt.Errorf("send: %w", &url.Error{Op: "Get", URL: "http://api.example", Err: context.DeadlineExceeded})
164+
out := processor.Process(err)
165+
assert.True(t, errs.IsRetryable(out))
166+
assert.True(t, errs.IsDependencyError(out), "should be attributed to the HTTP dependency")
167+
})
168+
158169
t.Run("a controller verdict wins over the classifier", func(t *testing.T) {
159170
// Pass 1 of the processor short-circuits on the existing framework wrap,
160171
// so a 502 a controller decided was fatal stays fatal.
@@ -164,3 +175,28 @@ func TestClassifier_AppliedViaProcessor(t *testing.T) {
164175
assert.False(t, errs.IsRetryable(out))
165176
})
166177
}
178+
179+
// TestClassifier_WithoutMySQLClassifier covers a service with no MySQL
180+
// dependency: no verdict here may rely on mysqlerrs' net.Error rule.
181+
func TestClassifier_WithoutMySQLClassifier(t *testing.T) {
182+
processor := errs.NewClassifierProcessor(genericerrs.Classifier, Classifier)
183+
184+
tests := []struct {
185+
name string
186+
cause error
187+
wantDependency bool
188+
}{
189+
{name: "connection reset", cause: errors.New("connection reset by peer"), wantDependency: true},
190+
{name: "expired deadline", cause: context.DeadlineExceeded, wantDependency: true},
191+
{name: "our cancellation", cause: context.Canceled, wantDependency: false},
192+
}
193+
194+
for _, tt := range tests {
195+
t.Run(tt.name, func(t *testing.T) {
196+
err := fmt.Errorf("send: %w", &url.Error{Op: "Get", URL: "http://api.example", Err: tt.cause})
197+
out := processor.Process(err)
198+
assert.True(t, errs.IsRetryable(out), "must not depend on mysqlerrs being wired")
199+
assert.Equal(t, tt.wantDependency, errs.IsDependencyError(out))
200+
})
201+
}
202+
}

platform/extension/buildrunner/buildkite/client.go

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,11 @@ type BuildResponse struct {
6666
}
6767

6868
// CreateBuild creates a new Buildkite build.
69+
//
70+
// POST /builds is not idempotent and a rejection carries its status code like any
71+
// other, so a caller that retries a 502 can create a second build when the first
72+
// was already accepted. Callers that cannot tolerate that need their own
73+
// idempotency check.
6974
func (c *Client) CreateBuild(ctx context.Context, req CreateBuildRequest) (BuildResponse, error) {
7075
body, err := json.Marshal(req)
7176
if err != nil {

platform/extension/buildrunner/buildkite/client_test.go

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,15 @@ func newTestClient(t *testing.T, handler http.Handler) *Client {
3838
return NewClient(c)
3939
}
4040

41+
// requireStatusError asserts err carries code as a *phttp.StatusError, the shape
42+
// platform/errs/http needs to classify it.
43+
func requireStatusError(t *testing.T, err error, code int) {
44+
t.Helper()
45+
var se *phttp.StatusError
46+
require.ErrorAs(t, err, &se)
47+
assert.Equal(t, code, se.StatusCode)
48+
}
49+
4150
func buildJSON(t *testing.T, number int, state, webURL string) []byte {
4251
t.Helper()
4352
return buildJSONWithEnv(t, number, state, webURL, nil)
@@ -86,6 +95,7 @@ func TestCreateBuild_ErrorStatus_ReturnsError(t *testing.T) {
8695

8796
_, err := c.CreateBuild(context.Background(), CreateBuildRequest{})
8897
require.Error(t, err)
98+
requireStatusError(t, err, http.StatusInternalServerError)
8999
}
90100

91101
// --- GetBuild ---
@@ -150,7 +160,9 @@ func TestCancelBuild_ErrorStatus_ReturnsError(t *testing.T) {
150160
w.WriteHeader(http.StatusInternalServerError)
151161
}))
152162

153-
require.Error(t, c.CancelBuild(context.Background(), 5))
163+
err := c.CancelBuild(context.Background(), 5)
164+
require.Error(t, err)
165+
requireStatusError(t, err, http.StatusInternalServerError)
154166
}
155167

156168
// --- EncodeBuildNumber / ParseBuildNumber ---

platform/extension/buildrunner/githubactions/client.go

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -106,6 +106,11 @@ type WorkflowRun struct {
106106
}
107107

108108
// DispatchWorkflow dispatches the bound workflow.
109+
//
110+
// Dispatching is not idempotent and a rejection carries its status code like any
111+
// other, so a caller that retries a 502 can start a second run when the first was
112+
// already accepted. Callers that cannot tolerate that need their own idempotency
113+
// check.
109114
func (c *Client) DispatchWorkflow(ctx context.Context, req DispatchWorkflowRequest) (DispatchWorkflowResponse, error) {
110115
body, err := json.Marshal(req)
111116
if err != nil {
@@ -146,7 +151,7 @@ func (c *Client) CancelRun(ctx context.Context, runID int64) error {
146151
case http.StatusNotFound:
147152
return ErrNotFound
148153
default:
149-
return fmt.Errorf("unexpected status %d from cancel", status)
154+
return fmt.Errorf("cancel run: %w", phttp.NewStatusError(status, nil))
150155
}
151156
}
152157

@@ -185,7 +190,9 @@ func (c *Client) do(ctx context.Context, method, rawURL string, body []byte, out
185190
return ErrNotFound
186191
}
187192
if status < 200 || status >= 300 {
188-
return fmt.Errorf("API returned status %d: %s", status, respBody)
193+
// Typed rather than formatted so platform/errs/http can read the code
194+
// and tell a transient 502 from a permanent 400.
195+
return phttp.NewStatusError(status, respBody)
189196
}
190197

191198
if out != nil && len(respBody) > 0 {

platform/extension/buildrunner/githubactions/client_test.go

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,15 @@ func newTestClient(t *testing.T, handler http.Handler) *Client {
4040
return NewClient(c, "uber", "submitqueue", "submitqueue-ci.yml")
4141
}
4242

43+
// requireStatusError asserts err carries code as a *phttp.StatusError, the shape
44+
// platform/errs/http needs to classify it.
45+
func requireStatusError(t *testing.T, err error, code int) {
46+
t.Helper()
47+
var se *phttp.StatusError
48+
require.ErrorAs(t, err, &se)
49+
assert.Equal(t, code, se.StatusCode)
50+
}
51+
4352
// --- NewClient / accessors ---
4453

4554
func TestNewClient_ExposesIdentity(t *testing.T) {
@@ -89,6 +98,7 @@ func TestDispatchWorkflow_ErrorStatus_ReturnsError(t *testing.T) {
8998

9099
_, err := c.DispatchWorkflow(context.Background(), DispatchWorkflowRequest{})
91100
require.Error(t, err)
101+
requireStatusError(t, err, http.StatusInternalServerError)
92102
}
93103

94104
// --- GetRun ---
@@ -147,6 +157,16 @@ func TestCancelRun_NotFound_ReturnsError(t *testing.T) {
147157
require.Error(t, c.CancelRun(context.Background(), 5))
148158
}
149159

160+
func TestCancelRun_ErrorStatus_ReturnsError(t *testing.T) {
161+
c := newTestClient(t, http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
162+
w.WriteHeader(http.StatusBadGateway)
163+
}))
164+
165+
err := c.CancelRun(context.Background(), 5)
166+
require.Error(t, err)
167+
requireStatusError(t, err, http.StatusBadGateway)
168+
}
169+
150170
// --- EncodeRunID / ParseRunID ---
151171

152172
func TestEncodeParseRunID_RoundTrip(t *testing.T) {

platform/http/status.go

Lines changed: 17 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,12 @@ package http
1616

1717
import "fmt"
1818

19+
// _maxRenderedBodyBytes bounds how much of Body reaches Error(). The rendered
20+
// string lands in a consumer's dead-letter record, whose column is finite, so an
21+
// error page from a chatty gateway must not be able to fail that write. Body
22+
// itself is kept whole.
23+
const _maxRenderedBodyBytes = 1024
24+
1925
// StatusError reports a response whose status code the caller rejected.
2026
//
2127
// SendRequest does not build this error itself: which codes count as success
@@ -35,7 +41,8 @@ type StatusError struct {
3541
// StatusCode is the HTTP status code from the response.
3642
StatusCode int
3743
// Body is the response body as read from the wire, or empty when the
38-
// caller had no body to attach.
44+
// caller had no body to attach. Error() renders at most
45+
// _maxRenderedBodyBytes of it.
3946
Body string
4047
}
4148

@@ -45,13 +52,17 @@ func NewStatusError(statusCode int, body []byte) *StatusError {
4552
return &StatusError{StatusCode: statusCode, Body: string(body)}
4653
}
4754

48-
// Error renders the status and, when present, the response body. Callers are
49-
// expected to wrap it with the operation that failed, giving messages like
50-
// "get build org/pipeline/builds/123: unexpected status 502: proxy forward
51-
// failed".
55+
// Error renders the status and, when present, the response body truncated to
56+
// _maxRenderedBodyBytes. Callers are expected to wrap it with the operation that
57+
// failed, giving messages like "get build org/pipeline/builds/123: unexpected
58+
// status 502: proxy forward failed".
5259
func (e *StatusError) Error() string {
5360
if e.Body == "" {
5461
return fmt.Sprintf("unexpected status %d", e.StatusCode)
5562
}
56-
return fmt.Sprintf("unexpected status %d: %s", e.StatusCode, e.Body)
63+
body := e.Body
64+
if len(body) > _maxRenderedBodyBytes {
65+
body = body[:_maxRenderedBodyBytes] + "… (truncated)"
66+
}
67+
return fmt.Sprintf("unexpected status %d: %s", e.StatusCode, body)
5768
}

platform/http/status_test.go

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,8 +15,10 @@
1515
package http
1616

1717
import (
18+
"bytes"
1819
"errors"
1920
"fmt"
21+
"strings"
2022
"testing"
2123

2224
"github.com/stretchr/testify/assert"
@@ -50,6 +52,17 @@ func TestNewStatusError(t *testing.T) {
5052
}
5153
}
5254

55+
func TestStatusError_RenderedBodyIsBounded(t *testing.T) {
56+
body := bytes.Repeat([]byte("a"), _maxRenderedBodyBytes*4)
57+
err := NewStatusError(502, body)
58+
59+
assert.Len(t, err.Body, _maxRenderedBodyBytes*4, "Body keeps what the caller passed")
60+
rendered := err.Error()
61+
assert.Less(t, len(rendered), _maxRenderedBodyBytes*2, "rendering must not grow with the body")
62+
assert.Contains(t, rendered, "(truncated)")
63+
assert.True(t, strings.HasPrefix(rendered, "unexpected status 502: aaa"))
64+
}
65+
5366
func TestStatusError_SurvivesWrapping(t *testing.T) {
5467
// Callers wrap with the operation that failed. The code has to stay
5568
// reachable through the chain, otherwise the classifier cannot read it.

0 commit comments

Comments
 (0)