fix(errs): Retry transient grouped failures

platform/errs/BUILD.bazel

@@ -5,6 +5,7 @@ go_library(
     srcs = [
         "errs.go",
         "failure.go",
+        "group.go",
         "processor.go",
     ],
     importpath = "github.com/uber/submitqueue/platform/errs",
@@ -17,6 +18,7 @@ go_test(
     srcs = [
         "errs_test.go",
         "failure_test.go",
+        "group_test.go",
         "processor_test.go",
     ],
     embed = [":go_default_library"],

platform/errs/README.md

@@ -44,13 +44,44 @@ An `ErrorProcessor` runs the per-chain pass that turns a raw chain into a wrappe
 Two implementations ship in this package:
 
 - **`NewClassifierProcessor(classifiers...)`** — the standard pass for primary pipeline consumers. Walks the chain twice:
-  1. **Pass 1 — framework-wrap check.** A cheap type switch looks for an existing `*userError` / `*infraError` anywhere in the chain. If found, the chain is already interpretable and the processor returns `err` unchanged. **No classifier is invoked.**
+  1. **Pass 1 — framework-wrap check.** Looks for an existing `*userError` / `*infraError` on the error's single-cause spine. If found, the chain is already interpretable and the processor returns `err` unchanged. **No classifier is invoked.**
   2. **Pass 2 — classifier walk.** From outermost to innermost node, each registered classifier is asked for a verdict. The first non-`Unknown` verdict wins and `err` is wrapped with the matching framework constructor.
 
   If no classifier recognises anything, `err` is returned unchanged — and behaves as non-retryable infra at the helper layer.
 
 - **`AlwaysRetryableProcessor`** — unconditionally wraps every non-nil error with `NewRetryableError`, overriding any inner framework wrap. Use it for narrowly-scoped consumers — typically DLQ reconciliation — that must redeliver on any failure because there is no further dead-letter destination. Side-effect: an inner `*infraError(dependency=true)` is masked by the outer `retryable=true` wrap, since `errors.As` matches the outermost `*infraError` first. This is acceptable for the intended DLQ use case where only `IsRetryable` drives transport behaviour; do not pair this processor with a primary pipeline consumer or genuine user errors will retry forever instead of reaching their DLQ.
 
+### Grouped errors
+
+`Group(errs...)` reports several failures that happened together as one error. It drops nils and returns nil when every member is nil, so a step that fans work out to independent handlers can accumulate failures in a loop and return the result directly:
+
+```go
+var failures []error
+for _, h := range handlers {
+    if err := h.Handle(ctx, event); err != nil {
+        failures = append(failures, fmt.Errorf("%s: %w", h.Name(), err))
+    }
+}
+return errs.Group(failures...)
+```
+
+Pass 2 descends into a group's members as well as into ordinary single-cause wraps; Pass 1 walks only the single-cause spine. Without that descent, `errors.Unwrap` returns nil for a group, so a walk built on it alone sees the group node and nothing beneath it, and every member goes unclassified.
+
+**Grouping is opt-in — only `Group` is weighed.** `Unwrap() []error` is also what `errors.Join` and `fmt.Errorf` with several `%w` produce, and there the extra causes are incidental: a cleanup failure hung off the real one, or context that happens to be an error. Weighing those would let an unrelated sibling decide retryability for the whole chain. `Group` is the one spelling that means "these failures are independent, rank them against each other", so the walk keys on that type and every other multi-cause error stays opaque — classified by its outermost recognisable node like any single-cause chain. A caller that wants its members ranked says so by returning `Group`, as `submitqueue/extension/validator/composite` does for its children.
+
+The two shapes combine differently, because they mean different things:
+
+- **Down a wrap chain**, the outermost verdict wins. A wrapper saw the error it wrapped and classified anyway, so it speaks with more knowledge than its cause.
+- **Across the members of a group**, nothing shadows anything. The members are independent failures reported together, and their order is the order the caller ran them in, not a precedence. They combine by rank, so the result cannot depend on which member failed first.
+
+The rank puts retryable above non-retryable, because the two mistakes cost differently: a wrong "retryable" spends a bounded retry budget and then dead-letters anyway, while a wrong "non-retryable" throws away a failure that would have cleared on its own. Within a retryability tier, the verdict that implicates this service outranks the one pointing elsewhere, so a partly-local failure is not reported as a pure dependency or user problem — that ordering only moves attribution, since every non-retryable verdict produces the same transport outcome. See `verdictRank` for the table.
+
+A framework wrap classifies the subtree beneath it and no further. Above a group it covers the whole group and Pass 1 returns the error verbatim, so no member is consulted. *Inside* a member it is one member's account of one failure, with no standing to classify the failures beside it — so it contributes its own verdict to the rank like any other member. That is what keeps a sibling's transient failure from being discarded by a member that happened to arrive pre-classified, and it also removes an ordering artifact: two wrapped members of differing retryability used to resolve by whichever one `errors.As` reached first.
+
+The losing member keeps its wrap in the chain, so `IsUserError` and `IsRetryable` can both report true for the same grouped error — one from a member, one from the outer wrap. Only the outer wrap drives the retry decision, the same way it does under `AlwaysRetryableProcessor`; `IsUserError` carries that precedence as a contract note, since a caller checking it before `IsRetryable` would drop the transient member.
+
+One operational consequence worth knowing before relying on any of this: **retrying a group re-runs everything.** The retry redelivers to every child, including the ones that succeeded, so children must be idempotent, and a child that fails persistently with a retryable-looking error (a decommissioned service returning connection-refused, say) will spend the whole retry budget on every message. Drop such a child rather than absorbing it.
+
 ### Choosing a processor
 
 - **Primary pipeline consumer** → `NewClassifierProcessor(...)`. Controllers' explicit `NewUserError` / `NewDependencyError` wraps must survive so user errors don't get retried, and unclassified backend errors must be inspected by the registered classifiers.
@@ -131,7 +162,7 @@ if err != nil {
 Two practical rules fall out of the short-circuit semantics:
 
 - **Wrap with a framework constructor as soon as the controller knows the right verdict.** Any wrap added later in the chain still wins, but wrapping early keeps the intent close to the decision.
-- **A wrap anywhere in the chain blocks all classifiers — including for nodes deeper than the wrap.** If you want a classifier to still get a look at the cause, do not wrap above it. (In practice this is rare: controllers wrap because they have the final answer.)
+- **A wrap blocks all classifiers beneath it, including for nodes deeper than the wrap.** If you want a classifier to still get a look at the cause, do not wrap above it. (In practice this is rare: controllers wrap because they have the final answer.) It does not block the sibling members of a group, which are classified and ranked independently.
 
 ### When *not* to classify in a controller
 

platform/errs/group.go

@@ -0,0 +1,57 @@
+// Copyright (c) 2026 Uber Technologies, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package errs
+
+import "strings"
+
+// Group reports several failures that happened together as one error, for a
+// step that must run every handler rather than stop at the first failure.
+//
+// nil arguments are dropped and Group returns nil when all of them are nil.
+// errors.Is and errors.As reach every error passed in.
+func Group(errs ...error) error {
+	members := make([]error, 0, len(errs))
+	for _, err := range errs {
+		if err != nil {
+			members = append(members, err)
+		}
+	}
+	if len(members) == 0 {
+		return nil
+	}
+	return &groupedError{members: members}
+}
+
+// groupedError is the error Group returns.
+type groupedError struct {
+	// members are the failures reported together, in the order given. Never
+	// empty, and never contains a nil.
+	members []error
+}
+
+// Error joins the member messages on one line, because these land in
+// structured logs where a multi-line message becomes several records.
+func (e *groupedError) Error() string {
+	msgs := make([]string, 0, len(e.members))
+	for _, m := range e.members {
+		msgs = append(msgs, m.Error())
+	}
+	return strings.Join(msgs, "; ")
+}
+
+// Unwrap returns the grouped failures for errors.Is/As compatibility.
+func (e *groupedError) Unwrap() []error {
+	return e.members
+}

platform/errs/group_test.go

@@ -0,0 +1,79 @@
+// Copyright (c) 2026 Uber Technologies, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package errs
+
+import (
+	"errors"
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestGroup_NoFailures(t *testing.T) {
+	tests := map[string][]error{
+		"no members": nil,
+		"single nil": {nil},
+		"all nil":    {nil, nil},
+	}
+
+	for name, members := range tests {
+		t.Run(name, func(t *testing.T) {
+			assert.NoError(t, Group(members...))
+		})
+	}
+}
+
+func TestGroup_DropsNilMembers(t *testing.T) {
+	a := errors.New("a")
+	b := errors.New("b")
+
+	out := Group(nil, a, nil, b, nil)
+
+	require.Error(t, out)
+	assert.Equal(t, "a; b", out.Error())
+}
+
+// Every member must stay reachable, because a member the caller cannot find is
+// also a member the classifier cannot weigh.
+func TestGroup_ReachesEveryMember(t *testing.T) {
+	sentinel := errors.New("sentinel")
+	other := errors.New("other")
+
+	out := Group(fmt.Errorf("child a: %w", sentinel), other)
+
+	assert.True(t, errors.Is(out, sentinel))
+	assert.True(t, errors.Is(out, other))
+	assert.False(t, errors.Is(out, errors.New("never grouped")))
+}
+
+func TestGroup_ErrorsAsFindsAMemberType(t *testing.T) {
+	out := Group(errors.New("plain"), NewUserError(errors.New("bad input")))
+
+	var ue *userError
+	require.True(t, errors.As(out, &ue))
+	assert.True(t, IsUserError(out))
+}
+
+// One log record per failed delivery, not one per member.
+func TestGroup_MessageIsSingleLine(t *testing.T) {
+	out := Group(errors.New("first"), errors.New("second"), errors.New("third"))
+
+	assert.NotContains(t, out.Error(), "\n")
+	for _, want := range []string{"first", "second", "third"} {
+		assert.Contains(t, out.Error(), want)
+	}
+}

platform/errs/processor.go

@@ -50,26 +50,31 @@ type ErrorProcessor interface {
 // Semantics of Process on the returned processor:
 //
 //   - nil in, nil out.
-//   - If err's chain already carries a framework classification (*userError
-//     or *infraError anywhere in the chain), returns err unchanged — the chain
-//     is already interpretable by IsUserError / IsRetryable /
-//     IsDependencyError.
+//   - If err carries a framework classification (*userError or *infraError) on
+//     its single-cause spine, returns err unchanged — the chain is already
+//     interpretable by IsUserError / IsRetryable / IsDependencyError.
 //   - Otherwise, walks the chain from outermost to innermost, asking each
 //     classifier per node. The FIRST non-Unknown verdict wins; the outermost
 //     such node determines the wrap. err is wrapped with the framework
 //     constructor matching that verdict (User -> NewUserError, InfraRetryable
 //     -> NewRetryableError, etc.) and the wrapped error is returned.
 //   - Verdict Infra means "non-retryable infra" — which is already the default
 //     behavior for an unwrapped chain, so no wrap is added.
-//   - If no classifier recognises anything, err is returned unchanged.
+//   - If no classifier recognizes anything, err is returned unchanged.
 //
-// Implementation: two passes over the chain. Pass 1 is a cheap type check
-// looking for an existing framework wrap and short-circuits if one is found —
-// no classifier is invoked. Pass 2 runs the configured classifiers per node.
-// Walking the chain is cheap relative to a classifier call, so this avoids
-// running classifiers whenever the chain is already classified deeper down.
+// Implementation: two passes over the chain. Pass 1 looks for an existing
+// framework wrap and short-circuits if one is found — no classifier is invoked.
+// Pass 2 runs the configured classifiers per node. Walking the chain is cheap
+// relative to a classifier call, so this avoids running classifiers whenever
+// the chain is already classified deeper down.
 //
-// Passing no classifiers is valid — the processor will still honour any
+// Pass 2 also descends into the members of a Group; Pass 1 walks the
+// single-cause spine only. Grouping is opt-in: a Group means "independent
+// failures, weigh them against each other", and no other multi-cause shape
+// (errors.Join, fmt.Errorf with several %w) claims that meaning, so those stay
+// opaque. classify documents how members combine.
+//
+// Passing no classifiers is valid — the processor will still honor any
 // framework wrap already in the chain and otherwise return err unchanged.
 //
 // NOTE: this central classifier model cannot disambiguate errors of the same
@@ -90,29 +95,19 @@ func (p classifierProcessor) Process(err error) error {
 		return nil
 	}
 
-	// Pass 1 — cheap framework-wrap check. If any node already carries a
-	// framework type, the chain is interpretable as-is and classifiers are
-	// never invoked.
+	// Pass 1 — framework-wrap check, along the single-cause spine only. A wrap
+	// found here sits above everything else in err, so it classifies the whole
+	// error and is returned verbatim. errors.Unwrap yields nothing for a Group,
+	// so a wrap inside a member never answers for its siblings; classify weighs
+	// it against them instead.
 	for cur := err; cur != nil; cur = errors.Unwrap(cur) {
-		switch cur.(type) {
-		case *userError, *infraError:
+		if wrapVerdict(cur) != Unknown {
 			return err
 		}
 	}
 
-	// Pass 2 — run classifiers per node from outermost to innermost. Stop at
-	// the first non-Unknown verdict.
-	var verdict Verdict
-	for cur := err; cur != nil && verdict == Unknown; cur = errors.Unwrap(cur) {
-		for _, c := range p.classifiers {
-			if v := c.Classify(cur); v != Unknown {
-				verdict = v
-				break
-			}
-		}
-	}
-
-	switch verdict {
+	// Pass 2 — classify the chain and wrap with the verdict it reaches.
+	switch p.classify(err) {
 	case User:
 		return NewUserError(err)
 	case InfraRetryable:
@@ -127,6 +122,102 @@ func (p classifierProcessor) Process(err error) error {
 	return err
 }
 
+// classify returns the verdict for err — from a framework wrap if it carries
+// one, otherwise from the configured classifiers — or Unknown when nothing
+// recognizes any node in it.
+//
+// The walk treats the two ways an error can contain another differently,
+// because they mean different things:
+//
+//   - Down a wrap chain (Unwrap() error) the outermost verdict wins. A wrapper
+//     saw the error it wrapped and chose to add context on top of it, so it
+//     speaks with more knowledge than its cause.
+//   - Across the members of a Group nothing shadows anything.
+//     The members are independent failures that happened to be reported
+//     together, and their order is the order the caller ran them in, not a
+//     precedence. They combine by verdictRank so that the result cannot depend
+//     on which member failed first.
+//
+// The second rule is why a framework wrap does not get the short-circuit it
+// gets on the spine: within a group it is one member's account of one failure,
+// with no standing to classify the failures beside it.
+func (p classifierProcessor) classify(err error) Verdict {
+	for cur := err; cur != nil; {
+		// A wrap is authoritative for everything it contains, so it answers for
+		// this subtree without consulting a classifier.
+		if v := wrapVerdict(cur); v != Unknown {
+			return v
+		}
+
+		for _, c := range p.classifiers {
+			if v := c.Classify(cur); v != Unknown {
+				return v
+			}
+		}
+
+		if group, ok := cur.(*groupedError); ok {
+			verdict := Unknown
+			for _, member := range group.Unwrap() {
+				if v := p.classify(member); verdictRank(v) > verdictRank(verdict) {
+					verdict = v
+				}
+			}
+			return verdict
+		}
+
+		cur = errors.Unwrap(cur)
+	}
+	return Unknown
+}
+
+func wrapVerdict(err error) Verdict {
+	switch e := err.(type) {
+	case *userError:
+		return User
+	case *infraError:
+		switch {
+		case e.retryable && e.dependency:
+			return InfraDependencyRetryable
+		case e.retryable:
+			return InfraRetryable
+		case e.dependency:
+			return InfraDependency
+		default:
+			return Infra
+		}
+	}
+	return Unknown
+}
+
+// verdictRank orders the members of a Group; the highest rank becomes the
+// group's verdict.
+//
+// A table, not a comparison on Verdict, whose constants are declaration-ordered:
+// InfraDependency exceeds InfraRetryable by value, so ranking by value would let
+// a non-retryable member discard a retryable sibling.
+//
+// Retryable ranks above non-retryable — a wrong "retryable" spends a bounded
+// retry budget, a wrong "non-retryable" strands a failure that would have
+// cleared. The non-retryable verdicts all dead-letter alike, so their order only
+// sets attribution, most actionable first: ours to fix, then the requester's,
+// then a dependency nobody here can act on.
+func verdictRank(v Verdict) int {
+	switch v {
+	case InfraRetryable:
+		return 5
+	case InfraDependencyRetryable:
+		return 4
+	case Infra:
+		return 3
+	case User:
+		return 2
+	case InfraDependency:
+		return 1
+	default: // Unknown
+		return 0
+	}
+}
+
 // AlwaysRetryableProcessor classifies every non-nil error as InfraRetryable by
 // wrapping it with NewRetryableError. The wrap is unconditional: an inner
 // *userError or non-retryable *infraError is overridden because errors.As