diff --git a/control-plane/internal/dashboard/agentendpoint.go b/control-plane/internal/dashboard/agentendpoint.go index 39a94a5..e0e4620 100644 --- a/control-plane/internal/dashboard/agentendpoint.go +++ b/control-plane/internal/dashboard/agentendpoint.go @@ -55,5 +55,64 @@ func (s *Server) agentEndpoint(w http.ResponseWriter, r *http.Request) { writeJSON(w, http.StatusNotFound, map[string]string{"error": "provider has no advertised agent endpoint"}) return } - writeJSON(w, http.StatusOK, map[string]string{"agent_endpoint": endpoint, "public_key": hex.EncodeToString(publicKey)}) + response := agentEndpointResponse{AgentEndpoint: endpoint, PublicKey: hex.EncodeToString(publicKey)} + if ingressMbps, egressMbps, ok := s.declaredBandwidth(ctx, providerID); ok { + response.BandwidthIngressMbps = ingressMbps + response.BandwidthEgressMbps = egressMbps + } + writeJSON(w, http.StatusOK, response) +} + +// agentEndpointResponse is GET /api/v1/agent-endpoint/{provider_id}'s +// response shape. The bandwidth fields are ADR-015 §5's "the provider's +// own declared ResourceCapability.Bandwidth," added for the Network +// Validator's MeasureBandwidth probe to score against -- omitempty (and +// therefore simply absent, not zero-valued) when declaredBandwidth has +// no fresh capability data for this provider, so a caller can tell +// "nothing to compare against yet" apart from "provider declared 0 +// Mbps" (see networkvalidator.AgentEndpoint's doc comment for how the +// caller-side type handles that same distinction). +type agentEndpointResponse struct { + AgentEndpoint string `json:"agent_endpoint"` + PublicKey string `json:"public_key"` + BandwidthIngressMbps int32 `json:"bandwidth_ingress_mbps,omitempty"` + BandwidthEgressMbps int32 `json:"bandwidth_egress_mbps,omitempty"` +} + +// declaredBandwidth reads a provider's most recently heartbeated declared +// bandwidth capacity (ResourceCapability.Bandwidth) from the same Redis +// heartbeat cache the overview endpoint's per-provider capability display +// already reads (openinfra:heartbeat:, decoded via the same +// structHeartbeat this package already uses) -- ADR-015 §5 calls for +// reading "the same live directory data the scheduler already uses," and +// agentmanager's live provider directory (what the scheduler reads) is +// itself backed by this identical heartbeat cache (see +// internal/agentmanager/directory.go's heartbeatKeyPrefix). Deliberately +// does not check the cache entry's TTL/freshness the way the overview +// endpoint's liveness display does: a declared capacity is closer to a +// slow-changing configuration value than a liveness signal, so serving a +// slightly stale declared figure is an acceptable simplification here, +// not a correctness bug -- ok is false only when there is no cached +// capability data to read at all. +func (s *Server) declaredBandwidth(ctx context.Context, providerID string) (ingressMbps, egressMbps int32, ok bool) { + if s.redis == nil { + return 0, 0, false + } + key := "openinfra:heartbeat:" + providerID + values, err := s.redis.HMGet(ctx, key, "payload").Result() + if err != nil || len(values) != 1 || values[0] == nil { + return 0, 0, false + } + payloadBytes, isString := values[0].(string) + if !isString { + return 0, 0, false + } + var payload structHeartbeat + if err := payload.Unmarshal([]byte(payloadBytes)); err != nil || payload.ProviderID != providerID { + return 0, 0, false + } + if payload.Capabilities == nil || payload.Capabilities.Bandwidth == nil { + return 0, 0, false + } + return payload.Capabilities.Bandwidth.IngressMbps, payload.Capabilities.Bandwidth.EgressMbps, true } diff --git a/control-plane/internal/dashboard/agentendpoint_test.go b/control-plane/internal/dashboard/agentendpoint_test.go index a2de963..66b7174 100644 --- a/control-plane/internal/dashboard/agentendpoint_test.go +++ b/control-plane/internal/dashboard/agentendpoint_test.go @@ -1,10 +1,16 @@ package dashboard import ( + "context" "encoding/hex" "encoding/json" "net/http" + "os" "testing" + + sharedv1 "github.com/openinfra/network/protocol/generated/go/shared/v1" + "github.com/redis/go-redis/v9" + "google.golang.org/protobuf/proto" ) func TestAgentEndpointReturnsTheAdvertisedEndpointAndKey(t *testing.T) { @@ -65,6 +71,105 @@ func TestAgentEndpointReturnsNotFoundWhenNoEndpointWasEverAdvertised(t *testing. } } +// Without a configured Redis client (this test's server, like most of +// this package's, is built with a nil one -- see newAuthTestServer), +// declaredBandwidth must degrade to "nothing to report" rather than +// panicking on a nil-interface method call, and the response must simply +// omit the bandwidth fields (they're omitempty), not zero-value them. +func TestAgentEndpointOmitsBandwidthFieldsWithoutARedisClient(t *testing.T) { + ctx, server, pool := newAuthTestServer(t) + handler := server.Handler() + + if _, err := pool.Exec(ctx, ` + INSERT INTO providers (provider_id, public_key, protocol_version, agent_version, capabilities, status, registered_at, agent_endpoint) + VALUES ('provider-no-redis', $1, '1', 'test', $2, 2, now(), 'https://agent.example.invalid:50052')`, + make([]byte, 32), []byte{}, + ); err != nil { + t.Fatal(err) + } + + recorder := doJSON(t, handler, http.MethodGet, "/api/v1/agent-endpoint/provider-no-redis", nil) + if recorder.Code != http.StatusOK { + t.Fatalf("status = %d, body = %s", recorder.Code, recorder.Body.String()) + } + var decoded map[string]any + if err := json.Unmarshal(recorder.Body.Bytes(), &decoded); err != nil { + t.Fatal(err) + } + if _, present := decoded["bandwidth_ingress_mbps"]; present { + t.Fatalf("expected no bandwidth_ingress_mbps field without a Redis client, got %v", decoded) + } + if _, present := decoded["bandwidth_egress_mbps"]; present { + t.Fatalf("expected no bandwidth_egress_mbps field without a Redis client, got %v", decoded) + } +} + +// With a real Redis-backed heartbeat cache entry present (ADR-015 §5's +// "the same live directory data the scheduler already uses"), the +// agent-endpoint response must surface the provider's declared +// ingress/egress bandwidth -- MeasureBandwidth's tolerance check +// (control-plane/internal/networkvalidator) depends on this field +// actually being populated, not just present-but-zero. +func TestAgentEndpointIncludesDeclaredBandwidthFromTheHeartbeatCache(t *testing.T) { + redisURL := os.Getenv("OPENINFRA_TEST_REDIS_URL") + if redisURL == "" { + t.Skip("OPENINFRA_TEST_REDIS_URL is not set") + } + options, err := redis.ParseURL(redisURL) + if err != nil { + t.Fatal(err) + } + client := redis.NewClient(options) + t.Cleanup(func() { _ = client.Close() }) + + ctx, server, pool := newAuthTestServer(t) + server.redis = client + handler := server.Handler() + + const providerID = "provider-with-declared-bandwidth" + if _, err := pool.Exec(ctx, ` + INSERT INTO providers (provider_id, public_key, protocol_version, agent_version, capabilities, status, registered_at, agent_endpoint) + VALUES ($1, $2, '1', 'test', $3, 2, now(), 'https://agent.example.invalid:50052')`, + providerID, make([]byte, 32), []byte{}, + ); err != nil { + t.Fatal(err) + } + + payload := &heartbeatPayload{ + ProviderId: providerID, + Capabilities: &sharedv1.ResourceCapability{ + Bandwidth: &sharedv1.Bandwidth{IngressMbps: 750, EgressMbps: 400}, + }, + } + encoded, err := proto.Marshal(payload) + if err != nil { + t.Fatal(err) + } + heartbeatKey := "openinfra:heartbeat:" + providerID + if err := client.HSet(ctx, heartbeatKey, "payload", encoded).Err(); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = client.Del(context.Background(), heartbeatKey).Err() }) + + recorder := doJSON(t, handler, http.MethodGet, "/api/v1/agent-endpoint/"+providerID, nil) + if recorder.Code != http.StatusOK { + t.Fatalf("status = %d, body = %s", recorder.Code, recorder.Body.String()) + } + var response struct { + BandwidthIngressMbps int32 `json:"bandwidth_ingress_mbps"` + BandwidthEgressMbps int32 `json:"bandwidth_egress_mbps"` + } + if err := json.Unmarshal(recorder.Body.Bytes(), &response); err != nil { + t.Fatal(err) + } + if response.BandwidthIngressMbps != 750 { + t.Fatalf("bandwidth_ingress_mbps = %d, want 750", response.BandwidthIngressMbps) + } + if response.BandwidthEgressMbps != 400 { + t.Fatalf("bandwidth_egress_mbps = %d, want 400", response.BandwidthEgressMbps) + } +} + func TestAgentEndpointRejectsAnOversizedProviderID(t *testing.T) { _, server, _ := newAuthTestServer(t) handler := server.Handler() diff --git a/control-plane/internal/networkvalidator/bandwidth.go b/control-plane/internal/networkvalidator/bandwidth.go new file mode 100644 index 0000000..f3b3716 --- /dev/null +++ b/control-plane/internal/networkvalidator/bandwidth.go @@ -0,0 +1,230 @@ +package networkvalidator + +import ( + "bytes" + "context" + "crypto/ed25519" + "crypto/rand" + "crypto/sha256" + "encoding/binary" + "fmt" + "time" + + "github.com/google/uuid" + agentv1 "github.com/openinfra/network/protocol/generated/go/agent/v1" +) + +// bandwidthProbeDomain must byte-for-byte match agent-api's +// BANDWIDTH_PROBE_DOMAIN constant (provider-agent/crates/agent-api/ +// src/lib.rs) -- re-derived directly from that source, not assumed from +// a summary, matching challengeDomain's identical discipline above: a +// mismatch here silently breaks every MeasureBandwidth signature +// verification. +var bandwidthProbeDomain = []byte("openinfra-bandwidth-probe-v1\x00") + +// bandwidthProbeBytes is the size of the random upload payload this +// validator sends in a MeasureBandwidth probe, and also the size it +// requests back in the download direction (symmetric by default: ADR-015 +// does not require symmetry, but there is no a priori reason for this +// validator to probe one direction harder than the other). Matches +// agent-api's MAX_BANDWIDTH_PROBE_BYTES exactly (the largest probe +// agent-api will accept) -- see that constant's doc comment for the +// "large enough to time meaningfully, small enough to bound the Agent's +// per-request cost" reasoning ADR-015 §3 documents; using the maximum +// bound maximizes this probe's timing resolution. +const bandwidthProbeBytes = 8 * 1024 * 1024 + +// bandwidthMessageSizeLimit raises grpc-go's default 4 MiB per-message +// limit (see dial's doc comment) to accommodate a bandwidthProbeBytes- +// sized payload plus a fixed margin for protobuf/gRPC framing overhead +// beyond the raw payload bytes -- matching agent-api's own raised +// server-side limit (BANDWIDTH_MESSAGE_SIZE_LIMIT in agent-cli/src/ +// main.rs) so neither side rejects the other's message first. +const bandwidthMessageSizeLimit = bandwidthProbeBytes + 64*1024 + +// bandwidthToleranceBps (ADR-015 §5): measured throughput must reach at +// least this fraction (basis points, matching this codebase's existing +// 0..10_000 convention elsewhere, e.g. pallet-network-validator's +// score_bps) of a provider's own declared bandwidth in *both* directions +// to pass. 70% -- ADR-015 §5's own suggested value, adopted here as the +// actual threshold: generous enough to absorb this measurement's +// documented coarseness (one HTTP/2 stream, no multi-connection +// saturation, no TCP-slow-start removal, and estimateThroughputMbps's own +// proportional time-split approximation below), tight enough to still +// catch a materially false declaration, which is this dimension's +// explicit bar (ADR-015 §5's "honest caveat") -- not a precise SLA check. +const bandwidthToleranceBps = 7000 + +// MeasureBandwidth is ADR-015's replacement for a generic SolveChallenge +// call on the Network dimension: one round trip that sends and requests a +// large random payload in both directions, verifies the Agent's signed +// response, estimates ingress/egress throughput, and scores pass/fail +// against the provider's own declared ResourceCapability.Bandwidth +// (endpoint.DeclaredIngressMbps/DeclaredEgressMbps, sourced from the +// dashboard's agent-endpoint discovery response). Like Challenge, a +// non-nil error means discovery/probe construction could not even be +// attempted; every attempted-but-failed outcome (unreachable Agent, bad +// hash, bad signature, under-tolerance measurement) is a ChallengeResult +// with ScoreBps == 0 and a human-readable Reason, never a non-nil error. +func (c *ChallengeClient) MeasureBandwidth(ctx context.Context, providerID string) (ChallengeResult, error) { + endpoint, err := c.config.Resolver.Resolve(ctx, providerID) + if err != nil { + return ChallengeResult{}, fmt.Errorf("resolve agent endpoint for provider %s: %w", providerID, err) + } + + uploadPayload := make([]byte, bandwidthProbeBytes) + if _, err := rand.Read(uploadPayload); err != nil { + return ChallengeResult{}, fmt.Errorf("generate bandwidth probe payload: %w", err) + } + uploadPayloadHash := sha256.Sum256(uploadPayload) + probeID := uuid.NewString() + + callCtx, cancel := context.WithTimeout(ctx, c.config.DialTimeout+c.config.ChallengeTimeout) + defer cancel() + + response, elapsed, err := c.callMeasureBandwidth(callCtx, endpoint, probeID, uploadPayload, bandwidthProbeBytes) + if err != nil { + // No response exists to derive ADR-015 §5's SHA256(upload_ + // payload_hash ++ download_payload) payload_hash from -- falling + // back to a hash of what this validator sent so a failed attempt + // still carries *some* addressable evidence, not an all-zero + // hash. + return ChallengeResult{ScoreBps: failingScoreBps, SampleCount: 1, PayloadHash: uploadPayloadHash, Reason: err.Error()}, nil + } + + // ADR-015 §5's payload_hash formula is response-dependent (it hashes + // what the Agent returned, not what this validator sent), so it's + // computed once a response exists at all, before any verification + // step below -- every subsequent return, pass or fail, reports the + // same hash, matching what was actually received. + payloadHash := sha256.Sum256(append(append([]byte{}, response.UploadPayloadHash...), response.DownloadPayload...)) + + if response.ProbeId != probeID { + return ChallengeResult{ScoreBps: failingScoreBps, SampleCount: 1, PayloadHash: payloadHash, Reason: "response probe_id does not match the request"}, nil + } + if len(response.UploadPayloadHash) != sha256.Size { + return ChallengeResult{ScoreBps: failingScoreBps, SampleCount: 1, PayloadHash: payloadHash, Reason: "response upload_payload_hash has an unexpected length"}, nil + } + if !bytes.Equal(response.UploadPayloadHash, uploadPayloadHash[:]) { + return ChallengeResult{ScoreBps: failingScoreBps, SampleCount: 1, PayloadHash: payloadHash, Reason: "response upload_payload_hash does not match SHA256(upload_payload)"}, nil + } + if len(response.DownloadPayload) != bandwidthProbeBytes { + return ChallengeResult{ScoreBps: failingScoreBps, SampleCount: 1, PayloadHash: payloadHash, Reason: "response download_payload length does not match requested_download_bytes"}, nil + } + signed := bandwidthSignedBytes(response.ProbeId, response.UploadPayloadHash, response.DownloadPayload, response.ServerProcessingMs) + if !ed25519.Verify(endpoint.PublicKey, signed, response.Signature) { + return ChallengeResult{ScoreBps: failingScoreBps, SampleCount: 1, PayloadHash: payloadHash, Reason: "signature verification failed"}, nil + } + + ingressMbps, egressMbps := estimateThroughputMbps(elapsed, response.ServerProcessingMs, len(uploadPayload), len(response.DownloadPayload)) + ingressPasses := passesBandwidthTolerance(ingressMbps, endpoint.DeclaredIngressMbps) + egressPasses := passesBandwidthTolerance(egressMbps, endpoint.DeclaredEgressMbps) + if !ingressPasses || !egressPasses { + reason := fmt.Sprintf( + "measured throughput below tolerance: ingress=%.1fMbps (declared %dMbps), egress=%.1fMbps (declared %dMbps)", + ingressMbps, endpoint.DeclaredIngressMbps, egressMbps, endpoint.DeclaredEgressMbps, + ) + return ChallengeResult{ScoreBps: failingScoreBps, SampleCount: 1, PayloadHash: payloadHash, Reason: reason}, nil + } + return ChallengeResult{ + ScoreBps: passingScoreBps, SampleCount: 1, PayloadHash: payloadHash, + Reason: fmt.Sprintf("ok: ingress=%.1fMbps egress=%.1fMbps", ingressMbps, egressMbps), + }, nil +} + +// callMeasureBandwidth dials endpoint.AgentEndpoint (same mTLS trust +// model as callSolveChallenge, see dial's doc comment) and issues one +// MeasureBandwidth RPC, returning the validator's own wall-clock +// measurement of the whole round trip alongside the response -- +// estimateThroughputMbps needs both. +func (c *ChallengeClient) callMeasureBandwidth(ctx context.Context, endpoint AgentEndpoint, probeID string, uploadPayload []byte, requestedDownloadBytes uint32) (*agentv1.MeasureBandwidthResponse, time.Duration, error) { + connection, err := c.dial(ctx, endpoint) + if err != nil { + return nil, 0, err + } + defer connection.Close() + + client := agentv1.NewProviderAgentServiceClient(connection) + requestCtx, requestCancel := context.WithTimeout(ctx, c.config.ChallengeTimeout) + defer requestCancel() + + started := time.Now() + response, err := client.MeasureBandwidth(requestCtx, &agentv1.MeasureBandwidthRequest{ + ProbeId: probeID, + UploadPayload: uploadPayload, + RequestedDownloadBytes: requestedDownloadBytes, + }) + elapsed := time.Since(started) + if err != nil { + return nil, 0, fmt.Errorf("MeasureBandwidth RPC: %w", err) + } + return response, elapsed, nil +} + +// estimateThroughputMbps is ADR-015 §2's approximation, spelled out: a +// single unary gRPC call over the standard client API does not expose +// "time spent only writing the request" separately from "time spent only +// reading the response," so there is no way to time each direction +// exactly from this side. Instead: subtract the Agent's own reported +// server_processing_ms from the validator's wall-clock round trip to get +// the network-attributable portion, then split that time between the two +// directions proportionally to each direction's byte count, then convert +// each direction's slice of time into Mbps. ADR-015 §2/§5 name this exact +// class of coarseness as the deliberate MVP tradeoff, not an oversight. +func estimateThroughputMbps(elapsed time.Duration, serverProcessingMs uint32, uploadBytes, downloadBytes int) (ingressMbps, egressMbps float64) { + networkMs := elapsed.Seconds()*1000 - float64(serverProcessingMs) + if networkMs < 1 { + // Guards against clock skew/rounding producing a non-positive + // duration (e.g. a near-instant loopback round trip) -- 1ms is a + // floor, not a measurement. + networkMs = 1 + } + totalBytes := uploadBytes + downloadBytes + if totalBytes == 0 { + return 0, 0 + } + if uploadBytes > 0 { + uploadMs := networkMs * float64(uploadBytes) / float64(totalBytes) + ingressMbps = float64(uploadBytes) * 8 / uploadMs / 1000 + } + if downloadBytes > 0 { + downloadMs := networkMs * float64(downloadBytes) / float64(totalBytes) + egressMbps = float64(downloadBytes) * 8 / downloadMs / 1000 + } + return ingressMbps, egressMbps +} + +// passesBandwidthTolerance applies ADR-015 §5's tolerance check for one +// direction. A non-positive declaredMbps means the dashboard had no +// fresh declared-capacity data for this provider (see +// AgentEndpoint.DeclaredIngressMbps/DeclaredEgressMbps's doc comment) -- +// indistinguishable from an honest "declared 0 Mbps," but in either case +// there is nothing to catch as a false declaration, so this trivially +// passes rather than failing a provider for a gap in discovery data. +func passesBandwidthTolerance(measuredMbps float64, declaredMbps int32) bool { + if declaredMbps <= 0 { + return true + } + threshold := float64(declaredMbps) * float64(bandwidthToleranceBps) / 10000 + return measuredMbps >= threshold +} + +// bandwidthSignedBytes reproduces exactly what agent-api's +// measure_bandwidth handler signs (provider-agent/crates/agent-api/ +// src/lib.rs's bandwidth_signed_bytes): +// +// BANDWIDTH_PROBE_DOMAIN ++ be_u32(len(probe_id)) ++ probe_id +// ++ upload_payload_hash (32 bytes, fixed) +// ++ be_u32(len(download_payload)) ++ download_payload +// ++ be_u32(server_processing_ms) +func bandwidthSignedBytes(probeID string, uploadPayloadHash []byte, downloadPayload []byte, serverProcessingMs uint32) []byte { + signed := make([]byte, 0, len(bandwidthProbeDomain)+4+len(probeID)+len(uploadPayloadHash)+4+len(downloadPayload)+4) + signed = append(signed, bandwidthProbeDomain...) + signed = binary.BigEndian.AppendUint32(signed, uint32(len(probeID))) + signed = append(signed, probeID...) + signed = append(signed, uploadPayloadHash...) + signed = binary.BigEndian.AppendUint32(signed, uint32(len(downloadPayload))) + signed = append(signed, downloadPayload...) + signed = binary.BigEndian.AppendUint32(signed, serverProcessingMs) + return signed +} diff --git a/control-plane/internal/networkvalidator/bandwidth_test.go b/control-plane/internal/networkvalidator/bandwidth_test.go new file mode 100644 index 0000000..bcd5b59 --- /dev/null +++ b/control-plane/internal/networkvalidator/bandwidth_test.go @@ -0,0 +1,198 @@ +package networkvalidator + +import ( + "context" + "crypto/ed25519" + "crypto/rand" + "encoding/hex" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + "time" +) + +// TestMeasureBandwidthPassesWhenThroughputComfortablyExceedsTolerance +// probes a real loopback gRPC server: any real network trivially clears a +// 1 Mbps declared figure, so this deterministically exercises the +// passing path without needing to control real transfer timing. +func TestMeasureBandwidthPassesWhenThroughputComfortablyExceedsTolerance(t *testing.T) { + _, agentPriv, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + t.Fatalf("generate agent key: %v", err) + } + harness := startTestAgentHarness(t, &fakeAgentServer{ + privateKey: agentPriv, + declaredIngressMbps: 1, + declaredEgressMbps: 1, + }) + defer harness.close() + + client := newChallengeClient(t, harness) + result, err := client.MeasureBandwidth(context.Background(), harness.providerID) + if err != nil { + t.Fatalf("MeasureBandwidth: %v", err) + } + if result.ScoreBps != passingScoreBps { + t.Fatalf("ScoreBps = %d, want %d (reason=%q)", result.ScoreBps, passingScoreBps, result.Reason) + } + if result.SampleCount != 1 { + t.Fatalf("SampleCount = %d, want 1", result.SampleCount) + } +} + +// TestMeasureBandwidthFailsWhenThroughputWellUnderTolerance declares an +// astronomically high bandwidth figure (petabits/sec) that no real test +// environment's loopback link can plausibly clear 70% of -- a +// deterministic way to exercise the failing path without needing to +// simulate a genuinely slow link. +func TestMeasureBandwidthFailsWhenThroughputWellUnderTolerance(t *testing.T) { + _, agentPriv, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + t.Fatalf("generate agent key: %v", err) + } + harness := startTestAgentHarness(t, &fakeAgentServer{ + privateKey: agentPriv, + declaredIngressMbps: 100_000_000, // 100 Pbps + declaredEgressMbps: 100_000_000, + }) + defer harness.close() + + client := newChallengeClient(t, harness) + result, err := client.MeasureBandwidth(context.Background(), harness.providerID) + if err != nil { + t.Fatalf("MeasureBandwidth: %v", err) + } + if result.ScoreBps != failingScoreBps { + t.Fatalf("ScoreBps = %d, want %d (reason=%q)", result.ScoreBps, failingScoreBps, result.Reason) + } + if result.Reason == "" { + t.Fatal("expected a non-empty failure reason") + } +} + +func TestMeasureBandwidthFailsOnTamperedUploadHash(t *testing.T) { + _, agentPriv, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + t.Fatalf("generate agent key: %v", err) + } + harness := startTestAgentHarness(t, &fakeAgentServer{ + privateKey: agentPriv, + declaredIngressMbps: 1, // low enough that tolerance alone would pass + declaredEgressMbps: 1, + tamperBandwidthUploadHash: true, + }) + defer harness.close() + + client := newChallengeClient(t, harness) + result, err := client.MeasureBandwidth(context.Background(), harness.providerID) + if err != nil { + t.Fatalf("MeasureBandwidth: %v", err) + } + if result.ScoreBps != failingScoreBps { + t.Fatalf("ScoreBps = %d, want %d for a tampered upload_payload_hash", result.ScoreBps, failingScoreBps) + } +} + +func TestMeasureBandwidthFailsOnTamperedSignature(t *testing.T) { + _, agentPriv, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + t.Fatalf("generate agent key: %v", err) + } + harness := startTestAgentHarness(t, &fakeAgentServer{ + privateKey: agentPriv, + declaredIngressMbps: 1, + declaredEgressMbps: 1, + tamperBandwidthSignature: true, + }) + defer harness.close() + + client := newChallengeClient(t, harness) + result, err := client.MeasureBandwidth(context.Background(), harness.providerID) + if err != nil { + t.Fatalf("MeasureBandwidth: %v", err) + } + if result.ScoreBps != failingScoreBps { + t.Fatalf("ScoreBps = %d, want %d for a tampered signature", result.ScoreBps, failingScoreBps) + } +} + +func TestMeasureBandwidthReportsFailureForUnreachableAgent(t *testing.T) { + // A dashboard that resolves to a closed port -- the Agent is + // unreachable, which must score 0 with a reason, not return an error + // from MeasureBandwidth itself (see its doc comment). + dashboard := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _ = json.NewEncoder(w).Encode(map[string]any{ + "agent_endpoint": "https://127.0.0.1:1", // reserved, nothing listens here + "public_key": hex.EncodeToString(make([]byte, 32)), + "bandwidth_ingress_mbps": int32(1), + "bandwidth_egress_mbps": int32(1), + }) + })) + defer dashboard.Close() + + resolver, err := NewEndpointResolver(dashboard.URL) + if err != nil { + t.Fatalf("new endpoint resolver: %v", err) + } + client := NewChallengeClient(ChallengeClientConfig{ + Resolver: resolver, + ClientCertificate: testValidatorClientCert(t), + DialTimeout: 1 * time.Second, + ChallengeTimeout: 1 * time.Second, + }) + result, err := client.MeasureBandwidth(context.Background(), "irrelevant") + if err != nil { + t.Fatalf("MeasureBandwidth: %v", err) + } + if result.ScoreBps != failingScoreBps { + t.Fatalf("ScoreBps = %d, want %d for an unreachable agent", result.ScoreBps, failingScoreBps) + } + if result.Reason == "" { + t.Fatal("expected a non-empty failure reason") + } +} + +// passesBandwidthTolerance and estimateThroughputMbps are pure and cheap +// to unit-test directly, independent of any real network round trip. +func TestPassesBandwidthToleranceTreatsNonPositiveDeclaredAsAutoPass(t *testing.T) { + if !passesBandwidthTolerance(0, 0) { + t.Fatal("expected a non-positive declared figure to trivially pass") + } + if !passesBandwidthTolerance(0, -5) { + t.Fatal("expected a negative declared figure to trivially pass") + } +} + +func TestPassesBandwidthToleranceAppliesTheConfiguredFraction(t *testing.T) { + // 70% of 100 Mbps is 70 Mbps. + if !passesBandwidthTolerance(70, 100) { + t.Fatal("expected exactly the threshold to pass") + } + if passesBandwidthTolerance(69.9, 100) { + t.Fatal("expected just under the threshold to fail") + } +} + +func TestEstimateThroughputMbpsSplitsProportionallyBySize(t *testing.T) { + // 1 second of network time, symmetric byte counts -> each direction + // gets ~half the time and each direction's Mbps reflects its own byte + // count over that half. + ingress, egress := estimateThroughputMbps(1*time.Second, 0, 1_000_000, 1_000_000) + if ingress <= 0 || egress <= 0 { + t.Fatalf("expected positive throughput in both directions, got ingress=%v egress=%v", ingress, egress) + } + if ingress != egress { + t.Fatalf("expected symmetric byte counts to produce equal throughput, got ingress=%v egress=%v", ingress, egress) + } + + // All bytes in one direction: the other direction must be exactly 0, + // not merely small. + ingressOnly, egressOnly := estimateThroughputMbps(1*time.Second, 0, 1_000_000, 0) + if ingressOnly <= 0 { + t.Fatal("expected positive ingress throughput") + } + if egressOnly != 0 { + t.Fatalf("expected exactly zero egress throughput when downloadBytes is 0, got %v", egressOnly) + } +} diff --git a/control-plane/internal/networkvalidator/challenge.go b/control-plane/internal/networkvalidator/challenge.go index 074a85e..50c48f3 100644 --- a/control-plane/internal/networkvalidator/challenge.go +++ b/control-plane/internal/networkvalidator/challenge.go @@ -189,6 +189,32 @@ func (c *ChallengeClient) Challenge(ctx context.Context, providerID string, dime // authentication, and is called out here loudly per this task's explicit // instructions rather than shipped silently. func (c *ChallengeClient) callSolveChallenge(ctx context.Context, endpoint AgentEndpoint, challengeID string, challengeType agentv1.SolveChallengeRequest_Type, payload []byte) (*agentv1.SolveChallengeResponse, error) { + connection, err := c.dial(ctx, endpoint) + if err != nil { + return nil, err + } + defer connection.Close() + + client := agentv1.NewProviderAgentServiceClient(connection) + requestCtx, requestCancel := context.WithTimeout(ctx, c.config.ChallengeTimeout) + defer requestCancel() + response, err := client.SolveChallenge(requestCtx, &agentv1.SolveChallengeRequest{ + ChallengeId: challengeID, + Type: challengeType, + Payload: payload, + }) + if err != nil { + return nil, fmt.Errorf("SolveChallenge RPC: %w", err) + } + return response, nil +} + +// dial establishes the mTLS gRPC connection callSolveChallenge and (ADR- +// 015) callMeasureBandwidth both dial identically -- factored out purely +// to avoid duplicating the trust-model logic documented above between +// the two call sites; the security discussion in callSolveChallenge's +// doc comment applies unchanged to every caller of dial. +func (c *ChallengeClient) dial(ctx context.Context, endpoint AgentEndpoint) (*grpc.ClientConn, error) { parsed, err := url.Parse(endpoint.AgentEndpoint) if err != nil || parsed.Scheme != "https" || parsed.Host == "" { return nil, fmt.Errorf("agent endpoint %q is not a valid https address", endpoint.AgentEndpoint) @@ -208,24 +234,27 @@ func (c *ChallengeClient) callSolveChallenge(ctx context.Context, endpoint Agent dialCtx, cancel := context.WithTimeout(ctx, c.config.DialTimeout) defer cancel() - connection, err := grpc.DialContext(dialCtx, parsed.Host, grpc.WithTransportCredentials(credentials.NewTLS(tlsConfig)), grpc.WithBlock()) + connection, err := grpc.DialContext( + dialCtx, parsed.Host, + grpc.WithTransportCredentials(credentials.NewTLS(tlsConfig)), + grpc.WithBlock(), + // grpc-go's default per-message limit (4 MiB, both directions) + // predates MeasureBandwidth (ADR-015) and would reject its whole + // premise -- applied here (not just on the MeasureBandwidth call + // site) since one dialed connection is shared by every RPC this + // client issues, and every other RPC's messages are far smaller + // than this limit regardless. Matches agent-api's own raised + // server-side limit exactly (see agent-cli/src/main.rs's + // BANDWIDTH_MESSAGE_SIZE_LIMIT). + grpc.WithDefaultCallOptions( + grpc.MaxCallRecvMsgSize(bandwidthMessageSizeLimit), + grpc.MaxCallSendMsgSize(bandwidthMessageSizeLimit), + ), + ) if err != nil { return nil, fmt.Errorf("dial agent %s: %w", parsed.Host, err) } - defer connection.Close() - - client := agentv1.NewProviderAgentServiceClient(connection) - requestCtx, requestCancel := context.WithTimeout(ctx, c.config.ChallengeTimeout) - defer requestCancel() - response, err := client.SolveChallenge(requestCtx, &agentv1.SolveChallengeRequest{ - ChallengeId: challengeID, - Type: challengeType, - Payload: payload, - }) - if err != nil { - return nil, fmt.Errorf("SolveChallenge RPC: %w", err) - } - return response, nil + return connection, nil } // signedChallengeBytes reproduces exactly what agent-api's solve_challenge diff --git a/control-plane/internal/networkvalidator/challenge_test.go b/control-plane/internal/networkvalidator/challenge_test.go index f060131..a659e80 100644 --- a/control-plane/internal/networkvalidator/challenge_test.go +++ b/control-plane/internal/networkvalidator/challenge_test.go @@ -16,6 +16,7 @@ import ( "net/http/httptest" "os" "path/filepath" + "sync" "testing" "time" @@ -25,23 +26,42 @@ import ( "google.golang.org/grpc/credentials" ) -// fakeAgentServer reproduces agent-api's real solve_challenge handler -// (provider-agent/crates/agent-api/src/lib.rs) exactly, in Go, so this -// test exercises the validator's actual verification logic against a -// byte-for-byte faithful re-implementation of what the real Agent -// computes -- not a stub that just echoes back whatever would make the -// test pass. +// fakeAgentServer reproduces agent-api's real solve_challenge and +// measure_bandwidth handlers (provider-agent/crates/agent-api/src/lib.rs) +// exactly, in Go, so tests exercise the validator's actual verification +// logic against a byte-for-byte faithful re-implementation of what the +// real Agent computes -- not a stub that just echoes back whatever would +// make the test pass. type fakeAgentServer struct { agentv1.UnimplementedProviderAgentServiceServer privateKey ed25519.PrivateKey // tamperSignature/tamperResult let individual tests corrupt an - // otherwise-correct response to prove the validator's verification - // actually rejects a bad response, not just accepts a good one. + // otherwise-correct SolveChallenge response to prove the validator's + // verification actually rejects a bad response, not just accepts a + // good one. tamperSignature bool tamperResult bool + // tamperBandwidthSignature/tamperBandwidthUploadHash do the same for + // MeasureBandwidth responses (ADR-015). + tamperBandwidthSignature bool + tamperBandwidthUploadHash bool + // declaredIngressMbps/declaredEgressMbps feed the fake dashboard's + // agent-endpoint response (see startTestAgentHarness) -- the + // "provider's own declared bandwidth" ADR-015 §5's tolerance check + // scores a MeasureBandwidth probe against. + declaredIngressMbps int32 + declaredEgressMbps int32 + + mu sync.Mutex + solveChallengeTypes []agentv1.SolveChallengeRequest_Type + measureBandwidthCalls int } func (s *fakeAgentServer) SolveChallenge(_ context.Context, req *agentv1.SolveChallengeRequest) (*agentv1.SolveChallengeResponse, error) { + s.mu.Lock() + s.solveChallengeTypes = append(s.solveChallengeTypes, req.Type) + s.mu.Unlock() + sum := sha256.Sum256(req.Payload) result := sum[:] signed := signedChallengeBytes(req.ChallengeId, int32(req.Type), result) @@ -63,6 +83,40 @@ func (s *fakeAgentServer) SolveChallenge(_ context.Context, req *agentv1.SolveCh }, nil } +// MeasureBandwidth reproduces agent-api's real measure_bandwidth handler +// (see bandwidth.go's bandwidthSignedBytes doc comment for the exact +// signed-byte layout this mirrors). +func (s *fakeAgentServer) MeasureBandwidth(_ context.Context, req *agentv1.MeasureBandwidthRequest) (*agentv1.MeasureBandwidthResponse, error) { + s.mu.Lock() + s.measureBandwidthCalls++ + s.mu.Unlock() + + sum := sha256.Sum256(req.UploadPayload) + uploadPayloadHash := sum[:] + downloadPayload := make([]byte, req.RequestedDownloadBytes) + if _, err := rand.Read(downloadPayload); err != nil { + return nil, err + } + const serverProcessingMs = 1 + signed := bandwidthSignedBytes(req.ProbeId, uploadPayloadHash, downloadPayload, serverProcessingMs) + signature := ed25519.Sign(s.privateKey, signed) + if s.tamperBandwidthUploadHash { + uploadPayloadHash = append([]byte{}, uploadPayloadHash...) + uploadPayloadHash[0] ^= 0xFF + } + if s.tamperBandwidthSignature { + signature = append([]byte{}, signature...) + signature[0] ^= 0xFF + } + return &agentv1.MeasureBandwidthResponse{ + ProbeId: req.ProbeId, + UploadPayloadHash: uploadPayloadHash, + DownloadPayload: downloadPayload, + ServerProcessingMs: serverProcessingMs, + Signature: signature, + }, nil +} + // testAgentHarness wires up a real TLS-terminated gRPC server (the fake // Agent) plus an httptest dashboard server serving the agent-endpoint // discovery JSON, so ChallengeClient.Challenge is tested end to end: @@ -87,16 +141,28 @@ func startTestAgentHarness(t *testing.T, fake *fakeAgentServer) *testAgentHarnes ClientAuth: tls.RequireAnyClientCert, MinVersion: tls.VersionTLS13, } - grpcServer := grpc.NewServer(grpc.Creds(credentials.NewTLS(tlsConfig))) + // MaxRecvMsgSize/MaxSendMsgSize: this fake server stands in for the + // real Agent, which (agent-cli/src/main.rs) raises tonic's default + // 4 MiB message limit for MeasureBandwidth's bandwidthProbeBytes-sized + // payloads -- matched here so this harness exercises the same + // message-size behavior the real Agent has, not grpc-go's smaller + // default. + grpcServer := grpc.NewServer( + grpc.Creds(credentials.NewTLS(tlsConfig)), + grpc.MaxRecvMsgSize(bandwidthMessageSizeLimit), + grpc.MaxSendMsgSize(bandwidthMessageSizeLimit), + ) agentv1.RegisterProviderAgentServiceServer(grpcServer, fake) go func() { _ = grpcServer.Serve(listener) }() publicKey := fake.privateKey.Public().(ed25519.PublicKey) providerID := "test-provider-id" dashboard := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - _ = json.NewEncoder(w).Encode(map[string]string{ - "agent_endpoint": "https://" + listener.Addr().String(), - "public_key": hex.EncodeToString(publicKey), + _ = json.NewEncoder(w).Encode(map[string]any{ + "agent_endpoint": "https://" + listener.Addr().String(), + "public_key": hex.EncodeToString(publicKey), + "bandwidth_ingress_mbps": fake.declaredIngressMbps, + "bandwidth_egress_mbps": fake.declaredEgressMbps, }) })) diff --git a/control-plane/internal/networkvalidator/endpoint.go b/control-plane/internal/networkvalidator/endpoint.go index 73b8ad4..12c756a 100644 --- a/control-plane/internal/networkvalidator/endpoint.go +++ b/control-plane/internal/networkvalidator/endpoint.go @@ -22,6 +22,18 @@ const endpointLookupTimeout = 5 * time.Second type AgentEndpoint struct { AgentEndpoint string PublicKey ed25519.PublicKey + // DeclaredIngressMbps/DeclaredEgressMbps are ADR-015 §5's "the + // provider's own declared ResourceCapability.Bandwidth" -- the figure + // MeasureBandwidth's measured throughput is scored against. Both are + // 0 when the dashboard has no fresh heartbeat capability data for + // this provider yet (see internal/dashboard/agentendpoint.go's + // declaredBandwidth) -- indistinguishable here from "provider + // declared 0 Mbps"; MeasureBandwidth's tolerance check treats a + // non-positive declared figure as "nothing to verify against" either + // way, so this ambiguity does not silently fail a provider that + // simply hasn't heartbeated its capability yet. + DeclaredIngressMbps int32 + DeclaredEgressMbps int32 } // EndpointResolver looks up a provider's Agent network address and @@ -80,6 +92,13 @@ func (resolver *EndpointResolver) Resolve(ctx context.Context, providerID string var decoded struct { AgentEndpoint string `json:"agent_endpoint"` PublicKey string `json:"public_key"` + // Bandwidth fields are omitempty on the dashboard's side (absent + // entirely when it has no fresh heartbeat capability data for + // this provider), so they decode to their zero value here too -- + // see AgentEndpoint's doc comment for how that ambiguity is + // handled downstream. + BandwidthIngressMbps int32 `json:"bandwidth_ingress_mbps"` + BandwidthEgressMbps int32 `json:"bandwidth_egress_mbps"` } if err := json.Unmarshal(body, &decoded); err != nil { return AgentEndpoint{}, fmt.Errorf("decode agent-endpoint response: %w", err) @@ -91,5 +110,10 @@ func (resolver *EndpointResolver) Resolve(ctx context.Context, providerID string if decoded.AgentEndpoint == "" { return AgentEndpoint{}, fmt.Errorf("agent-endpoint discovery for provider %s returned an empty endpoint", providerID) } - return AgentEndpoint{AgentEndpoint: decoded.AgentEndpoint, PublicKey: publicKey}, nil + return AgentEndpoint{ + AgentEndpoint: decoded.AgentEndpoint, + PublicKey: publicKey, + DeclaredIngressMbps: decoded.BandwidthIngressMbps, + DeclaredEgressMbps: decoded.BandwidthEgressMbps, + }, nil } diff --git a/control-plane/internal/networkvalidator/run.go b/control-plane/internal/networkvalidator/run.go index eeb5369..07f1f01 100644 --- a/control-plane/internal/networkvalidator/run.go +++ b/control-plane/internal/networkvalidator/run.go @@ -173,7 +173,18 @@ func challengeAndSubmit(ctx context.Context, cfg LoopConfig, key roundKey, done logger := cfg.Logger.With("provider_id", providerID, "round", key.round, "dimension", key.dimension.String()) logger.Info("networkvalidator: assigned, challenging") - result, err := cfg.Challenger.Challenge(ctx, providerID, key.dimension) + // ADR-015: the Network dimension's evidence changes from a generic + // SolveChallenge liveness/correctness check to a real MeasureBandwidth + // throughput probe -- run instead of, not in addition to, + // Challenge()'s SolveChallenge call. Every other dimension is + // unchanged. + var result ChallengeResult + var err error + if key.dimension == blockchainbridge.DimensionNetwork { + result, err = cfg.Challenger.MeasureBandwidth(ctx, providerID) + } else { + result, err = cfg.Challenger.Challenge(ctx, providerID, key.dimension) + } if err != nil { // Discovery/setup could not even be attempted (e.g. dashboard // unreachable) -- do not mark done, so a later tick this same diff --git a/control-plane/internal/networkvalidator/run_test.go b/control-plane/internal/networkvalidator/run_test.go index b71cfd3..beb3566 100644 --- a/control-plane/internal/networkvalidator/run_test.go +++ b/control-plane/internal/networkvalidator/run_test.go @@ -20,6 +20,7 @@ import ( "time" "github.com/openinfra/network/internal/blockchainbridge" + agentv1 "github.com/openinfra/network/protocol/generated/go/agent/v1" "golang.org/x/crypto/blake2b" ) @@ -182,7 +183,8 @@ func TestRunEndToEndSubmitsEvidenceThenAttemptsCloseRound(t *testing.T) { if err != nil { t.Fatalf("generate agent key: %v", err) } - harness := startTestAgentHarness(t, &fakeAgentServer{privateKey: agentPriv}) + fake := &fakeAgentServer{privateKey: agentPriv} + harness := startTestAgentHarness(t, fake) defer harness.close() rpc, err := blockchainbridge.NewRPCClient(chainServer.URL, &http.Client{Timeout: 5 * time.Second}) @@ -267,6 +269,37 @@ func TestRunEndToEndSubmitsEvidenceThenAttemptsCloseRound(t *testing.T) { if evidenceCount != len(Dimensions) { t.Errorf("submit_evidence called %d times, want exactly %d (once per dimension, never re-submitted)", evidenceCount, len(Dimensions)) } + + // ADR-015: the Network dimension's evidence must come from exactly one + // MeasureBandwidth call, never a SolveChallenge(TYPE_NETWORK) call -- + // and every other dimension must still go through SolveChallenge + // exactly once, unchanged. + fake.mu.Lock() + measureBandwidthCalls := fake.measureBandwidthCalls + solveChallengeTypes := append([]agentv1.SolveChallengeRequest_Type{}, fake.solveChallengeTypes...) + fake.mu.Unlock() + + if measureBandwidthCalls != 1 { + t.Errorf("MeasureBandwidth called %d times, want exactly 1", measureBandwidthCalls) + } + wantSolveChallengeTypes := map[agentv1.SolveChallengeRequest_Type]int{ + agentv1.SolveChallengeRequest_TYPE_COMPUTE: 1, + agentv1.SolveChallengeRequest_TYPE_STORAGE: 1, + agentv1.SolveChallengeRequest_TYPE_AVAILABILITY: 1, + agentv1.SolveChallengeRequest_TYPE_RELIABILITY: 1, + } + gotSolveChallengeTypes := map[agentv1.SolveChallengeRequest_Type]int{} + for _, solveChallengeType := range solveChallengeTypes { + gotSolveChallengeTypes[solveChallengeType]++ + } + for solveChallengeType, wantCount := range wantSolveChallengeTypes { + if gotSolveChallengeTypes[solveChallengeType] != wantCount { + t.Errorf("SolveChallenge(%s) called %d times, want %d", solveChallengeType, gotSolveChallengeTypes[solveChallengeType], wantCount) + } + } + if count := gotSolveChallengeTypes[agentv1.SolveChallengeRequest_TYPE_NETWORK]; count != 0 { + t.Errorf("SolveChallenge(TYPE_NETWORK) called %d times, want 0 -- the Network dimension must use MeasureBandwidth instead (ADR-015)", count) + } } // loadRegistrarFromPrivateKey writes privateKey to a temp PKCS8 PEM file diff --git a/docs/adr/015-bandwidth-throughput-measurement.md b/docs/adr/015-bandwidth-throughput-measurement.md new file mode 100644 index 0000000..4d8a823 --- /dev/null +++ b/docs/adr/015-bandwidth-throughput-measurement.md @@ -0,0 +1,165 @@ +# ADR-015: Independent bandwidth throughput measurement + +## Status + +Accepted. + +## Context + +Issue #30's first slice (merged) made bandwidth a reserved, scheduled +resource — but the capacity a provider advertises is entirely +operator-declared, the same trust boundary as its price/reputation +self-declarations, with zero independent verification (issue #73). The +Network Validator daemon built for #78/ADR-013 does independently test +every `ScoreDimension` including `Network` — but `SolveChallenge`'s +existing protocol (`agent-api`'s `solve_challenge` handler, +`MAX_CHALLENGE_PAYLOAD = 4096` bytes) computes `SHA256(payload)` on a +small, bounded input and times the whole RPC round trip. That is a +liveness-and-correctness test — proof the Agent is reachable and signs +correctly within a deadline — not a throughput measurement. A 4 KB +payload transferred in a few seconds says essentially nothing about a +link's real Mbps capacity, especially on a fast local/dev network where +transfer time is dominated by RPC/TLS overhead, not payload size. + +`agent-api`'s `StreamMetrics` RPC exists in `agent.proto` but is entirely +unimplemented (`Status::unimplemented`) — not reusable as-is, and its +shape (an empty request, a stream of periodic metrics *from* the Agent) +doesn't fit an active bandwidth probe naturally; it reads as designed for +ongoing telemetry, not a bounded on-demand measurement. + +This ADR decides the measurement protocol only — see #73 for the larger, +explicitly out-of-scope remainder (WireGuard overhead accounting, +regional endpoint selection, workload-level rate limit enforcement, and +the adversarial test suite: congestion, asymmetric links, spoofed +results, partitions). + +## Decision + +### 1. A new, dedicated RPC — not an overload of `SolveChallenge`'s `TYPE_NETWORK` + +`TYPE_NETWORK` already exists, is merged, and is exercised by the live +challenge loop (#85) with its existing small-payload semantics. +Redefining what it measures now would silently change already-shipped, +tested behavior. Instead: a new RPC, +`rpc MeasureBandwidth(MeasureBandwidthRequest) returns (MeasureBandwidthResponse)`, +on `ProviderAgentService`. + +```protobuf +message MeasureBandwidthRequest { + string probe_id = 1; + // Sent by the validator; its size is what's actually being measured + // for *ingress* (validator -> Agent). Bounded server-side (see §3). + bytes upload_payload = 2; + // Requested size of download_payload in the response, for measuring + // *egress* (Agent -> validator) in the same round trip. 0 means "don't + // bother," e.g. when a caller only wants one direction this probe. + uint32 requested_download_bytes = 3; +} + +message MeasureBandwidthResponse { + string probe_id = 1; + bytes upload_payload_hash = 2; // SHA256(upload_payload) -- proves full, correct receipt + bytes download_payload = 3; // exactly requested_download_bytes of Agent-generated data + uint32 server_processing_ms = 4; // time from full request received to response ready, excluding queueing + bytes signature = 5; // Ed25519 over a domain-separated construction, see §4 +} +``` + +Both directions share one RPC deliberately: a validator that wants only +one direction sets the other side's size to (near) zero, and a single +round trip avoids the clock-skew problems of trying to reconcile two +separately-timed calls. + +### 2. What is actually measured, and by whom + +The **validator** times the whole RPC round trip on its own clock (dial +already established, so this is pure request-write + response-read +time), and separately reads `server_processing_ms` from the response to +subtract server-side compute/serialization from the network-bound +portion it cares about. Two throughput figures per probe: + +- `ingress_mbps ≈ len(upload_payload) * 8 / (upload_write_time_ms) / 1000` + (approximated from the portion of the round trip attributable to + sending — see §5's honest caveat about why this is approximate, not + exact). +- `egress_mbps ≈ len(download_payload) * 8 / (download_read_time_ms) / 1000`. + +This is deliberately coarse (one HTTP/2 stream, no multi-connection +saturation, no removal of TCP slow-start effects) — a real ISP-grade +bandwidth test (iperf3-style) is a much larger undertaking than an MVP +validator daemon needs. The goal is "plausibly close to the declared +figure, done independently, better than trusting the operator outright," +not a laboratory-grade measurement. + +### 3. Payload size and rate limiting + +`upload_payload`/`requested_download_bytes` are both bounded by a new +constant, `MAX_BANDWIDTH_PROBE_BYTES` (agent-api), distinct from and +larger than `MAX_CHALLENGE_PAYLOAD` — large enough to produce a +meaningful timing signal over a local/typical WAN link (an 8 MiB probe +takes ~7ms at 10 Gbps, ~640ms at 100 Mbps — enough resolution to +distinguish realistic tiers without either flooding a slow link for +unreasonably long or being too short to time meaningfully on a fast one), +small enough to bound the Agent's per-request memory/CPU cost the same +way every other Agent RPC already is bounded. `agent-api` rate-limits +this RPC per caller (reuses the existing allowlist-authenticated-caller +identity from ADR-013 §3 -- a validator is already an authenticated +caller by the time it can call any Agent RPC) to prevent a validator +(malicious or buggy) from using repeated large probes as a bandwidth- +exhaustion vector against a provider it does not like. + +### 4. Signing and evidence + +`signature` covers a domain-separated construction analogous to +`solve_challenge`'s: `BANDWIDTH_PROBE_DOMAIN ++ probe_id ++ +upload_payload_hash ++ download_payload ++ be_u32(server_processing_ms)` +(exact byte layout is the implementing PR's responsibility to pin down +and document precisely, matching this ADR's intent, not necessarily this +exact byte order). The validator verifies this the same way it already +verifies `SolveChallenge` responses (Ed25519, using the Agent's public +key from the existing agent-endpoint discovery response), so a +fabricated/tampered measurement is still caught by the same signature- +verification discipline `Challenge()` already applies. + +### 5. Scoring: same binary philosophy, against the declared figure + +`Network` dimension evidence for a round becomes: run one +`MeasureBandwidth` probe, compute `ingress_mbps`/`egress_mbps`, compare +each against the provider's own declared `ResourceCapability.Bandwidth` +(read from the same live directory data the scheduler already uses) with +a tolerance factor (a new constant, e.g. 70%: measured must reach at +least 70% of declared in *both* directions to pass) — matching the +existing binary pass/fail convention (`score_bps = 10_000` or `0`, +`sample_count = 1`) every other dimension in the challenge loop already +uses, not a new latency-graded scheme. `payload_hash` for the evidence +submission is `SHA256(upload_payload_hash ++ download_payload)`, a +bounded, addressable summary of what was actually measured. + +**Honest caveat, stated plainly rather than hidden:** a single-stream, +single-probe-per-round measurement over gRPC/HTTP2 is a real but +approximate signal, not a certified benchmark. It is still strictly more +than #30's first slice had (zero independent verification at all), and +is explicitly scoped as "good enough to catch a materially false +declaration," not "precise enough to bill by." + +## Consequences + +- A new Agent RPC (`MeasureBandwidth`) is new attack surface: needs its + own payload-size bound, its own rate limit, and its own tests + (oversized payload rejected, malformed `probe_id` rejected, signature + verifies against the Agent's real key, an unauthenticated/non- + allowlisted caller is rejected the same way any other Agent RPC already + is via ADR-013 §3's mTLS trust model). +- The challenge loop's `Network` dimension evidence changes from a + liveness/correctness check to a real throughput measurement -- + behavior change to already-shipped code (#85), done deliberately and + documented here, not silently. +- Still explicitly out of scope, tracked in #73: WireGuard overhead + accounting (a lease-gated overlay adds its own throughput ceiling this + ADR's raw Agent-to-validator measurement doesn't see), regional + endpoint selection, workload-level rate limit *enforcement* (this ADR + only measures capacity, it does not throttle a running workload's + actual usage against its reservation), and the adversarial test suite + (congestion, asymmetric links, spoofed results, partitions) -- each is + its own substantial piece, not a quick follow-up to this measurement + protocol. diff --git a/protocol/generated/go/agent/v1/agent.pb.go b/protocol/generated/go/agent/v1/agent.pb.go index 32b0ba9..83f79e3 100644 --- a/protocol/generated/go/agent/v1/agent.pb.go +++ b/protocol/generated/go/agent/v1/agent.pb.go @@ -939,6 +939,152 @@ func (x *StreamMetricsResponse) GetTimestamp() int64 { return 0 } +// ADR-015 §1. Both directions share one round trip deliberately: a caller +// that wants only one direction sets the other side's size to (near) +// zero, avoiding the clock-skew problems of reconciling two separately +// timed calls. +type MeasureBandwidthRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + ProbeId string `protobuf:"bytes,1,opt,name=probe_id,json=probeId,proto3" json:"probe_id,omitempty"` + // Sent by the validator; its size is what's actually being measured + // for *ingress* (validator -> Agent). Bounded server-side by + // agent-api's MAX_BANDWIDTH_PROBE_BYTES. + UploadPayload []byte `protobuf:"bytes,2,opt,name=upload_payload,json=uploadPayload,proto3" json:"upload_payload,omitempty"` + // Requested size of download_payload in the response, for measuring + // *egress* (Agent -> validator) in the same round trip. 0 means "don't + // bother," e.g. when a caller only wants one direction this probe. + RequestedDownloadBytes uint32 `protobuf:"varint,3,opt,name=requested_download_bytes,json=requestedDownloadBytes,proto3" json:"requested_download_bytes,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *MeasureBandwidthRequest) Reset() { + *x = MeasureBandwidthRequest{} + mi := &file_openinfra_agent_v1_agent_proto_msgTypes[14] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *MeasureBandwidthRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*MeasureBandwidthRequest) ProtoMessage() {} + +func (x *MeasureBandwidthRequest) ProtoReflect() protoreflect.Message { + mi := &file_openinfra_agent_v1_agent_proto_msgTypes[14] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use MeasureBandwidthRequest.ProtoReflect.Descriptor instead. +func (*MeasureBandwidthRequest) Descriptor() ([]byte, []int) { + return file_openinfra_agent_v1_agent_proto_rawDescGZIP(), []int{14} +} + +func (x *MeasureBandwidthRequest) GetProbeId() string { + if x != nil { + return x.ProbeId + } + return "" +} + +func (x *MeasureBandwidthRequest) GetUploadPayload() []byte { + if x != nil { + return x.UploadPayload + } + return nil +} + +func (x *MeasureBandwidthRequest) GetRequestedDownloadBytes() uint32 { + if x != nil { + return x.RequestedDownloadBytes + } + return 0 +} + +type MeasureBandwidthResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + ProbeId string `protobuf:"bytes,1,opt,name=probe_id,json=probeId,proto3" json:"probe_id,omitempty"` + UploadPayloadHash []byte `protobuf:"bytes,2,opt,name=upload_payload_hash,json=uploadPayloadHash,proto3" json:"upload_payload_hash,omitempty"` // SHA256(upload_payload) -- proves full, correct receipt + DownloadPayload []byte `protobuf:"bytes,3,opt,name=download_payload,json=downloadPayload,proto3" json:"download_payload,omitempty"` // exactly requested_download_bytes of Agent-generated data + ServerProcessingMs uint32 `protobuf:"varint,4,opt,name=server_processing_ms,json=serverProcessingMs,proto3" json:"server_processing_ms,omitempty"` // time from full request received to response ready, excluding queueing + Signature []byte `protobuf:"bytes,5,opt,name=signature,proto3" json:"signature,omitempty"` // Ed25519 over a domain-separated construction, see agent-api's BANDWIDTH_PROBE_DOMAIN + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *MeasureBandwidthResponse) Reset() { + *x = MeasureBandwidthResponse{} + mi := &file_openinfra_agent_v1_agent_proto_msgTypes[15] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *MeasureBandwidthResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*MeasureBandwidthResponse) ProtoMessage() {} + +func (x *MeasureBandwidthResponse) ProtoReflect() protoreflect.Message { + mi := &file_openinfra_agent_v1_agent_proto_msgTypes[15] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use MeasureBandwidthResponse.ProtoReflect.Descriptor instead. +func (*MeasureBandwidthResponse) Descriptor() ([]byte, []int) { + return file_openinfra_agent_v1_agent_proto_rawDescGZIP(), []int{15} +} + +func (x *MeasureBandwidthResponse) GetProbeId() string { + if x != nil { + return x.ProbeId + } + return "" +} + +func (x *MeasureBandwidthResponse) GetUploadPayloadHash() []byte { + if x != nil { + return x.UploadPayloadHash + } + return nil +} + +func (x *MeasureBandwidthResponse) GetDownloadPayload() []byte { + if x != nil { + return x.DownloadPayload + } + return nil +} + +func (x *MeasureBandwidthResponse) GetServerProcessingMs() uint32 { + if x != nil { + return x.ServerProcessingMs + } + return 0 +} + +func (x *MeasureBandwidthResponse) GetSignature() []byte { + if x != nil { + return x.Signature + } + return nil +} + type HealthCheckRequest struct { state protoimpl.MessageState `protogen:"open.v1"` unknownFields protoimpl.UnknownFields @@ -947,7 +1093,7 @@ type HealthCheckRequest struct { func (x *HealthCheckRequest) Reset() { *x = HealthCheckRequest{} - mi := &file_openinfra_agent_v1_agent_proto_msgTypes[14] + mi := &file_openinfra_agent_v1_agent_proto_msgTypes[16] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -959,7 +1105,7 @@ func (x *HealthCheckRequest) String() string { func (*HealthCheckRequest) ProtoMessage() {} func (x *HealthCheckRequest) ProtoReflect() protoreflect.Message { - mi := &file_openinfra_agent_v1_agent_proto_msgTypes[14] + mi := &file_openinfra_agent_v1_agent_proto_msgTypes[16] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -972,7 +1118,7 @@ func (x *HealthCheckRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use HealthCheckRequest.ProtoReflect.Descriptor instead. func (*HealthCheckRequest) Descriptor() ([]byte, []int) { - return file_openinfra_agent_v1_agent_proto_rawDescGZIP(), []int{14} + return file_openinfra_agent_v1_agent_proto_rawDescGZIP(), []int{16} } type HealthCheckResponse struct { @@ -985,7 +1131,7 @@ type HealthCheckResponse struct { func (x *HealthCheckResponse) Reset() { *x = HealthCheckResponse{} - mi := &file_openinfra_agent_v1_agent_proto_msgTypes[15] + mi := &file_openinfra_agent_v1_agent_proto_msgTypes[17] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -997,7 +1143,7 @@ func (x *HealthCheckResponse) String() string { func (*HealthCheckResponse) ProtoMessage() {} func (x *HealthCheckResponse) ProtoReflect() protoreflect.Message { - mi := &file_openinfra_agent_v1_agent_proto_msgTypes[15] + mi := &file_openinfra_agent_v1_agent_proto_msgTypes[17] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1010,7 +1156,7 @@ func (x *HealthCheckResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use HealthCheckResponse.ProtoReflect.Descriptor instead. func (*HealthCheckResponse) Descriptor() ([]byte, []int) { - return file_openinfra_agent_v1_agent_proto_rawDescGZIP(), []int{15} + return file_openinfra_agent_v1_agent_proto_rawDescGZIP(), []int{17} } func (x *HealthCheckResponse) GetHealthy() bool { @@ -1035,7 +1181,7 @@ type GetAgentInfoRequest struct { func (x *GetAgentInfoRequest) Reset() { *x = GetAgentInfoRequest{} - mi := &file_openinfra_agent_v1_agent_proto_msgTypes[16] + mi := &file_openinfra_agent_v1_agent_proto_msgTypes[18] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1047,7 +1193,7 @@ func (x *GetAgentInfoRequest) String() string { func (*GetAgentInfoRequest) ProtoMessage() {} func (x *GetAgentInfoRequest) ProtoReflect() protoreflect.Message { - mi := &file_openinfra_agent_v1_agent_proto_msgTypes[16] + mi := &file_openinfra_agent_v1_agent_proto_msgTypes[18] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1060,7 +1206,7 @@ func (x *GetAgentInfoRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use GetAgentInfoRequest.ProtoReflect.Descriptor instead. func (*GetAgentInfoRequest) Descriptor() ([]byte, []int) { - return file_openinfra_agent_v1_agent_proto_rawDescGZIP(), []int{16} + return file_openinfra_agent_v1_agent_proto_rawDescGZIP(), []int{18} } var File_openinfra_agent_v1_agent_proto protoreflect.FileDescriptor @@ -1140,12 +1286,22 @@ const file_openinfra_agent_v1_agent_proto_rawDesc = "" + "\vmetric_name\x18\x01 \x01(\tR\n" + "metricName\x12\x14\n" + "\x05value\x18\x02 \x01(\x01R\x05value\x12\x1c\n" + - "\ttimestamp\x18\x03 \x01(\x03R\ttimestamp\"\x14\n" + + "\ttimestamp\x18\x03 \x01(\x03R\ttimestamp\"\x95\x01\n" + + "\x17MeasureBandwidthRequest\x12\x19\n" + + "\bprobe_id\x18\x01 \x01(\tR\aprobeId\x12%\n" + + "\x0eupload_payload\x18\x02 \x01(\fR\ruploadPayload\x128\n" + + "\x18requested_download_bytes\x18\x03 \x01(\rR\x16requestedDownloadBytes\"\xe0\x01\n" + + "\x18MeasureBandwidthResponse\x12\x19\n" + + "\bprobe_id\x18\x01 \x01(\tR\aprobeId\x12.\n" + + "\x13upload_payload_hash\x18\x02 \x01(\fR\x11uploadPayloadHash\x12)\n" + + "\x10download_payload\x18\x03 \x01(\fR\x0fdownloadPayload\x120\n" + + "\x14server_processing_ms\x18\x04 \x01(\rR\x12serverProcessingMs\x12\x1c\n" + + "\tsignature\x18\x05 \x01(\fR\tsignature\"\x14\n" + "\x12HealthCheckRequest\"G\n" + "\x13HealthCheckResponse\x12\x18\n" + "\ahealthy\x18\x01 \x01(\bR\ahealthy\x12\x16\n" + "\x06status\x18\x02 \x01(\tR\x06status\"\x15\n" + - "\x13GetAgentInfoRequest2\x9b\x06\n" + + "\x13GetAgentInfoRequest2\x8a\a\n" + "\x14ProviderAgentService\x12a\n" + "\fGetAgentInfo\x12'.openinfra.agent.v1.GetAgentInfoRequest\x1a(.openinfra.agent.v1.GetAgentInfoResponse\x12^\n" + "\vHealthCheck\x12&.openinfra.agent.v1.HealthCheckRequest\x1a'.openinfra.agent.v1.HealthCheckResponse\x12a\n" + @@ -1154,7 +1310,8 @@ const file_openinfra_agent_v1_agent_proto_rawDesc = "" + "\x06Deploy\x12!.openinfra.agent.v1.DeployRequest\x1a\".openinfra.agent.v1.DeployResponse\x12I\n" + "\x04Stop\x12\x1f.openinfra.agent.v1.StopRequest\x1a .openinfra.agent.v1.StopResponse\x12p\n" + "\x11GetWorkloadStatus\x12,.openinfra.agent.v1.GetWorkloadStatusRequest\x1a-.openinfra.agent.v1.GetWorkloadStatusResponse\x12f\n" + - "\rStreamMetrics\x12(.openinfra.agent.v1.StreamMetricsRequest\x1a).openinfra.agent.v1.StreamMetricsResponse0\x01BEZCgithub.com/openinfra/network/protocol/generated/go/agent/v1;agentv1b\x06proto3" + "\rStreamMetrics\x12(.openinfra.agent.v1.StreamMetricsRequest\x1a).openinfra.agent.v1.StreamMetricsResponse0\x01\x12m\n" + + "\x10MeasureBandwidth\x12+.openinfra.agent.v1.MeasureBandwidthRequest\x1a,.openinfra.agent.v1.MeasureBandwidthResponseBEZCgithub.com/openinfra/network/protocol/generated/go/agent/v1;agentv1b\x06proto3" var ( file_openinfra_agent_v1_agent_proto_rawDescOnce sync.Once @@ -1169,7 +1326,7 @@ func file_openinfra_agent_v1_agent_proto_rawDescGZIP() []byte { } var file_openinfra_agent_v1_agent_proto_enumTypes = make([]protoimpl.EnumInfo, 2) -var file_openinfra_agent_v1_agent_proto_msgTypes = make([]protoimpl.MessageInfo, 17) +var file_openinfra_agent_v1_agent_proto_msgTypes = make([]protoimpl.MessageInfo, 19) var file_openinfra_agent_v1_agent_proto_goTypes = []any{ (SolveChallengeRequest_Type)(0), // 0: openinfra.agent.v1.SolveChallengeRequest.Type (GetWorkloadStatusResponse_State)(0), // 1: openinfra.agent.v1.GetWorkloadStatusResponse.State @@ -1187,32 +1344,36 @@ var file_openinfra_agent_v1_agent_proto_goTypes = []any{ (*GetInventoryResponse)(nil), // 13: openinfra.agent.v1.GetInventoryResponse (*StreamMetricsRequest)(nil), // 14: openinfra.agent.v1.StreamMetricsRequest (*StreamMetricsResponse)(nil), // 15: openinfra.agent.v1.StreamMetricsResponse - (*HealthCheckRequest)(nil), // 16: openinfra.agent.v1.HealthCheckRequest - (*HealthCheckResponse)(nil), // 17: openinfra.agent.v1.HealthCheckResponse - (*GetAgentInfoRequest)(nil), // 18: openinfra.agent.v1.GetAgentInfoRequest + (*MeasureBandwidthRequest)(nil), // 16: openinfra.agent.v1.MeasureBandwidthRequest + (*MeasureBandwidthResponse)(nil), // 17: openinfra.agent.v1.MeasureBandwidthResponse + (*HealthCheckRequest)(nil), // 18: openinfra.agent.v1.HealthCheckRequest + (*HealthCheckResponse)(nil), // 19: openinfra.agent.v1.HealthCheckResponse + (*GetAgentInfoRequest)(nil), // 20: openinfra.agent.v1.GetAgentInfoRequest } var file_openinfra_agent_v1_agent_proto_depIdxs = []int32{ 0, // 0: openinfra.agent.v1.SolveChallengeRequest.type:type_name -> openinfra.agent.v1.SolveChallengeRequest.Type 6, // 1: openinfra.agent.v1.DeployRequest.limits:type_name -> openinfra.agent.v1.ResourceLimits 1, // 2: openinfra.agent.v1.GetWorkloadStatusResponse.state:type_name -> openinfra.agent.v1.GetWorkloadStatusResponse.State - 18, // 3: openinfra.agent.v1.ProviderAgentService.GetAgentInfo:input_type -> openinfra.agent.v1.GetAgentInfoRequest - 16, // 4: openinfra.agent.v1.ProviderAgentService.HealthCheck:input_type -> openinfra.agent.v1.HealthCheckRequest + 20, // 3: openinfra.agent.v1.ProviderAgentService.GetAgentInfo:input_type -> openinfra.agent.v1.GetAgentInfoRequest + 18, // 4: openinfra.agent.v1.ProviderAgentService.HealthCheck:input_type -> openinfra.agent.v1.HealthCheckRequest 12, // 5: openinfra.agent.v1.ProviderAgentService.GetInventory:input_type -> openinfra.agent.v1.GetInventoryRequest 3, // 6: openinfra.agent.v1.ProviderAgentService.SolveChallenge:input_type -> openinfra.agent.v1.SolveChallengeRequest 5, // 7: openinfra.agent.v1.ProviderAgentService.Deploy:input_type -> openinfra.agent.v1.DeployRequest 8, // 8: openinfra.agent.v1.ProviderAgentService.Stop:input_type -> openinfra.agent.v1.StopRequest 10, // 9: openinfra.agent.v1.ProviderAgentService.GetWorkloadStatus:input_type -> openinfra.agent.v1.GetWorkloadStatusRequest 14, // 10: openinfra.agent.v1.ProviderAgentService.StreamMetrics:input_type -> openinfra.agent.v1.StreamMetricsRequest - 2, // 11: openinfra.agent.v1.ProviderAgentService.GetAgentInfo:output_type -> openinfra.agent.v1.GetAgentInfoResponse - 17, // 12: openinfra.agent.v1.ProviderAgentService.HealthCheck:output_type -> openinfra.agent.v1.HealthCheckResponse - 13, // 13: openinfra.agent.v1.ProviderAgentService.GetInventory:output_type -> openinfra.agent.v1.GetInventoryResponse - 4, // 14: openinfra.agent.v1.ProviderAgentService.SolveChallenge:output_type -> openinfra.agent.v1.SolveChallengeResponse - 7, // 15: openinfra.agent.v1.ProviderAgentService.Deploy:output_type -> openinfra.agent.v1.DeployResponse - 9, // 16: openinfra.agent.v1.ProviderAgentService.Stop:output_type -> openinfra.agent.v1.StopResponse - 11, // 17: openinfra.agent.v1.ProviderAgentService.GetWorkloadStatus:output_type -> openinfra.agent.v1.GetWorkloadStatusResponse - 15, // 18: openinfra.agent.v1.ProviderAgentService.StreamMetrics:output_type -> openinfra.agent.v1.StreamMetricsResponse - 11, // [11:19] is the sub-list for method output_type - 3, // [3:11] is the sub-list for method input_type + 16, // 11: openinfra.agent.v1.ProviderAgentService.MeasureBandwidth:input_type -> openinfra.agent.v1.MeasureBandwidthRequest + 2, // 12: openinfra.agent.v1.ProviderAgentService.GetAgentInfo:output_type -> openinfra.agent.v1.GetAgentInfoResponse + 19, // 13: openinfra.agent.v1.ProviderAgentService.HealthCheck:output_type -> openinfra.agent.v1.HealthCheckResponse + 13, // 14: openinfra.agent.v1.ProviderAgentService.GetInventory:output_type -> openinfra.agent.v1.GetInventoryResponse + 4, // 15: openinfra.agent.v1.ProviderAgentService.SolveChallenge:output_type -> openinfra.agent.v1.SolveChallengeResponse + 7, // 16: openinfra.agent.v1.ProviderAgentService.Deploy:output_type -> openinfra.agent.v1.DeployResponse + 9, // 17: openinfra.agent.v1.ProviderAgentService.Stop:output_type -> openinfra.agent.v1.StopResponse + 11, // 18: openinfra.agent.v1.ProviderAgentService.GetWorkloadStatus:output_type -> openinfra.agent.v1.GetWorkloadStatusResponse + 15, // 19: openinfra.agent.v1.ProviderAgentService.StreamMetrics:output_type -> openinfra.agent.v1.StreamMetricsResponse + 17, // 20: openinfra.agent.v1.ProviderAgentService.MeasureBandwidth:output_type -> openinfra.agent.v1.MeasureBandwidthResponse + 12, // [12:21] is the sub-list for method output_type + 3, // [3:12] is the sub-list for method input_type 3, // [3:3] is the sub-list for extension type_name 3, // [3:3] is the sub-list for extension extendee 0, // [0:3] is the sub-list for field type_name @@ -1229,7 +1390,7 @@ func file_openinfra_agent_v1_agent_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_openinfra_agent_v1_agent_proto_rawDesc), len(file_openinfra_agent_v1_agent_proto_rawDesc)), NumEnums: 2, - NumMessages: 17, + NumMessages: 19, NumExtensions: 0, NumServices: 1, }, diff --git a/protocol/generated/go/agent/v1/agent_grpc.pb.go b/protocol/generated/go/agent/v1/agent_grpc.pb.go index 51e1cb9..a83cf91 100644 --- a/protocol/generated/go/agent/v1/agent_grpc.pb.go +++ b/protocol/generated/go/agent/v1/agent_grpc.pb.go @@ -27,6 +27,7 @@ const ( ProviderAgentService_Stop_FullMethodName = "/openinfra.agent.v1.ProviderAgentService/Stop" ProviderAgentService_GetWorkloadStatus_FullMethodName = "/openinfra.agent.v1.ProviderAgentService/GetWorkloadStatus" ProviderAgentService_StreamMetrics_FullMethodName = "/openinfra.agent.v1.ProviderAgentService/StreamMetrics" + ProviderAgentService_MeasureBandwidth_FullMethodName = "/openinfra.agent.v1.ProviderAgentService/MeasureBandwidth" ) // ProviderAgentServiceClient is the client API for ProviderAgentService service. @@ -41,6 +42,12 @@ type ProviderAgentServiceClient interface { Stop(ctx context.Context, in *StopRequest, opts ...grpc.CallOption) (*StopResponse, error) GetWorkloadStatus(ctx context.Context, in *GetWorkloadStatusRequest, opts ...grpc.CallOption) (*GetWorkloadStatusResponse, error) StreamMetrics(ctx context.Context, in *StreamMetricsRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[StreamMetricsResponse], error) + // ADR-015: independent bandwidth throughput measurement. A dedicated RPC, + // deliberately not an overload of SolveChallenge's existing TYPE_NETWORK + // (see ADR-015 §1) -- that type is already merged and exercised by the + // live challenge loop (#85) with its existing small-payload liveness/ + // correctness semantics, which this does not change. + MeasureBandwidth(ctx context.Context, in *MeasureBandwidthRequest, opts ...grpc.CallOption) (*MeasureBandwidthResponse, error) } type providerAgentServiceClient struct { @@ -140,6 +147,16 @@ func (c *providerAgentServiceClient) StreamMetrics(ctx context.Context, in *Stre // This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. type ProviderAgentService_StreamMetricsClient = grpc.ServerStreamingClient[StreamMetricsResponse] +func (c *providerAgentServiceClient) MeasureBandwidth(ctx context.Context, in *MeasureBandwidthRequest, opts ...grpc.CallOption) (*MeasureBandwidthResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(MeasureBandwidthResponse) + err := c.cc.Invoke(ctx, ProviderAgentService_MeasureBandwidth_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + // ProviderAgentServiceServer is the server API for ProviderAgentService service. // All implementations must embed UnimplementedProviderAgentServiceServer // for forward compatibility. @@ -152,6 +169,12 @@ type ProviderAgentServiceServer interface { Stop(context.Context, *StopRequest) (*StopResponse, error) GetWorkloadStatus(context.Context, *GetWorkloadStatusRequest) (*GetWorkloadStatusResponse, error) StreamMetrics(*StreamMetricsRequest, grpc.ServerStreamingServer[StreamMetricsResponse]) error + // ADR-015: independent bandwidth throughput measurement. A dedicated RPC, + // deliberately not an overload of SolveChallenge's existing TYPE_NETWORK + // (see ADR-015 §1) -- that type is already merged and exercised by the + // live challenge loop (#85) with its existing small-payload liveness/ + // correctness semantics, which this does not change. + MeasureBandwidth(context.Context, *MeasureBandwidthRequest) (*MeasureBandwidthResponse, error) mustEmbedUnimplementedProviderAgentServiceServer() } @@ -186,6 +209,9 @@ func (UnimplementedProviderAgentServiceServer) GetWorkloadStatus(context.Context func (UnimplementedProviderAgentServiceServer) StreamMetrics(*StreamMetricsRequest, grpc.ServerStreamingServer[StreamMetricsResponse]) error { return status.Error(codes.Unimplemented, "method StreamMetrics not implemented") } +func (UnimplementedProviderAgentServiceServer) MeasureBandwidth(context.Context, *MeasureBandwidthRequest) (*MeasureBandwidthResponse, error) { + return nil, status.Error(codes.Unimplemented, "method MeasureBandwidth not implemented") +} func (UnimplementedProviderAgentServiceServer) mustEmbedUnimplementedProviderAgentServiceServer() {} func (UnimplementedProviderAgentServiceServer) testEmbeddedByValue() {} @@ -344,6 +370,24 @@ func _ProviderAgentService_StreamMetrics_Handler(srv interface{}, stream grpc.Se // This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. type ProviderAgentService_StreamMetricsServer = grpc.ServerStreamingServer[StreamMetricsResponse] +func _ProviderAgentService_MeasureBandwidth_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(MeasureBandwidthRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ProviderAgentServiceServer).MeasureBandwidth(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: ProviderAgentService_MeasureBandwidth_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ProviderAgentServiceServer).MeasureBandwidth(ctx, req.(*MeasureBandwidthRequest)) + } + return interceptor(ctx, in, info, handler) +} + // ProviderAgentService_ServiceDesc is the grpc.ServiceDesc for ProviderAgentService service. // It's only intended for direct use with grpc.RegisterService, // and not to be introspected or modified (even as a copy) @@ -379,6 +423,10 @@ var ProviderAgentService_ServiceDesc = grpc.ServiceDesc{ MethodName: "GetWorkloadStatus", Handler: _ProviderAgentService_GetWorkloadStatus_Handler, }, + { + MethodName: "MeasureBandwidth", + Handler: _ProviderAgentService_MeasureBandwidth_Handler, + }, }, Streams: []grpc.StreamDesc{ { diff --git a/protocol/proto/openinfra/agent/v1/agent.proto b/protocol/proto/openinfra/agent/v1/agent.proto index ea17986..567fdd3 100644 --- a/protocol/proto/openinfra/agent/v1/agent.proto +++ b/protocol/proto/openinfra/agent/v1/agent.proto @@ -12,6 +12,12 @@ service ProviderAgentService { rpc Stop(StopRequest) returns (StopResponse); rpc GetWorkloadStatus(GetWorkloadStatusRequest) returns (GetWorkloadStatusResponse); rpc StreamMetrics(StreamMetricsRequest) returns (stream StreamMetricsResponse); + // ADR-015: independent bandwidth throughput measurement. A dedicated RPC, + // deliberately not an overload of SolveChallenge's existing TYPE_NETWORK + // (see ADR-015 §1) -- that type is already merged and exercised by the + // live challenge loop (#85) with its existing small-payload liveness/ + // correctness semantics, which this does not change. + rpc MeasureBandwidth(MeasureBandwidthRequest) returns (MeasureBandwidthResponse); } message GetAgentInfoResponse { @@ -110,6 +116,30 @@ message StreamMetricsResponse { int64 timestamp = 3; } +// ADR-015 §1. Both directions share one round trip deliberately: a caller +// that wants only one direction sets the other side's size to (near) +// zero, avoiding the clock-skew problems of reconciling two separately +// timed calls. +message MeasureBandwidthRequest { + string probe_id = 1; + // Sent by the validator; its size is what's actually being measured + // for *ingress* (validator -> Agent). Bounded server-side by + // agent-api's MAX_BANDWIDTH_PROBE_BYTES. + bytes upload_payload = 2; + // Requested size of download_payload in the response, for measuring + // *egress* (Agent -> validator) in the same round trip. 0 means "don't + // bother," e.g. when a caller only wants one direction this probe. + uint32 requested_download_bytes = 3; +} + +message MeasureBandwidthResponse { + string probe_id = 1; + bytes upload_payload_hash = 2; // SHA256(upload_payload) -- proves full, correct receipt + bytes download_payload = 3; // exactly requested_download_bytes of Agent-generated data + uint32 server_processing_ms = 4; // time from full request received to response ready, excluding queueing + bytes signature = 5; // Ed25519 over a domain-separated construction, see agent-api's BANDWIDTH_PROBE_DOMAIN +} + message HealthCheckRequest {} message HealthCheckResponse { bool healthy = 1; string status = 2; } message GetAgentInfoRequest {} diff --git a/provider-agent/Cargo.lock b/provider-agent/Cargo.lock index aaac905..d2b1233 100644 --- a/provider-agent/Cargo.lock +++ b/provider-agent/Cargo.lock @@ -10,15 +10,20 @@ dependencies = [ "agent-inventory", "anyhow", "async-trait", + "hex", "prost", "prost-types", "protoc-bin-vendored", + "rand", + "rcgen", "sha2", + "tempfile", "tokio", "tokio-stream", "tonic", "tonic-build", "tracing", + "x509-parser", ] [[package]] diff --git a/provider-agent/crates/agent-api/Cargo.toml b/provider-agent/crates/agent-api/Cargo.toml index 59f05af..1ca5097 100644 --- a/provider-agent/crates/agent-api/Cargo.toml +++ b/provider-agent/crates/agent-api/Cargo.toml @@ -15,6 +15,22 @@ tracing = "0.1" async-trait = "0.1" tokio-stream = "0.1" sha2 = "0.10" +rand = "0.8" +# Extracts a MeasureBandwidth caller's raw Ed25519 SubjectPublicKeyInfo +# from its mTLS leaf certificate (tonic::Request::peer_certs()) for the +# per-caller rate limiter -- a real X.509 parse, not a byte-offset guess, +# mirroring agent-cli/src/mtls.rs's identical extraction (duplicated, not +# shared, since agent-api cannot depend on agent-cli -- the dependency +# direction runs the other way). +x509-parser = "0.16" + +[dev-dependencies] +tempfile = "3" +hex = "0.4" +# Builds a real self-signed Ed25519 certificate to exercise the +# raw-public-key extraction helper against actual DER, the same fixture +# style agent-cli/src/mtls.rs's tests already use. +rcgen = "0.13" [build-dependencies] tonic-build = "0.10" diff --git a/provider-agent/crates/agent-api/src/lib.rs b/provider-agent/crates/agent-api/src/lib.rs index 2fcc580..8a8e2c3 100644 --- a/provider-agent/crates/agent-api/src/lib.rs +++ b/provider-agent/crates/agent-api/src/lib.rs @@ -22,8 +22,10 @@ use crate::proto::*; use agent_core::{identity::IdentityManager, local_state::LocalStateError, AgentConfig}; use agent_inventory::InventoryManager; use async_trait::async_trait; +use rand::RngCore; use sha2::{Digest, Sha256}; -use std::sync::Arc; +use std::collections::HashMap; +use std::sync::{Arc, Mutex}; use std::time::Duration; use std::time::Instant; use tokio::sync::{mpsc, oneshot}; @@ -45,6 +47,42 @@ const CHALLENGE_DOMAIN: &[u8] = b"openinfra-availability-proof-v1\0"; const MAX_CHALLENGE_ID: usize = 128; const MAX_CHALLENGE_PAYLOAD: usize = 4096; +// ADR-015 §1/§3: MeasureBandwidth's domain-separated signing constant and +// its payload-size bound. Reuses MAX_CHALLENGE_ID's convention for +// probe_id (same shape of field, same reasoning) but gets its own, much +// larger, payload bound -- MAX_CHALLENGE_PAYLOAD (4096 bytes) exists to +// bound a liveness/correctness proof-of-work input, not to produce a +// meaningful throughput timing signal. 8 MiB is large enough to time +// meaningfully even on a fast link (~7ms at 10 Gbps) and long enough to +// resolve realistic slower tiers (~640ms at 100 Mbps) without either +// flooding a slow link for an unreasonable duration or bounding the +// Agent's per-request memory/CPU cost any more loosely than every other +// Agent RPC already is bounded. +const BANDWIDTH_PROBE_DOMAIN: &[u8] = b"openinfra-bandwidth-probe-v1\0"; +// pub: agent-cli's server setup needs this value too, to raise tonic's +// default 4 MiB gRPC message-size limit (which would otherwise reject +// this RPC's whole premise) -- see main.rs's ProviderAgentServiceServer +// construction. Keeping one definition, referenced from both places, +// instead of a second hard-coded number that could silently drift from +// this one. +pub const MAX_BANDWIDTH_PROBE_BYTES: usize = 8 * 1024 * 1024; + +// ADR-015 §3: a per-caller rate limit scoped to just this RPC, so a +// validator (malicious or buggy) cannot use repeated MAX_BANDWIDTH_ +// PROBE_BYTES-sized probes as a bandwidth-exhaustion vector against a +// provider it does not like. A plain fixed-window counter (see +// BandwidthRateLimiter below), not a token bucket -- this is an MVP abuse +// bound, not a QoS system, and the codebase's other MVP-shortcut rate +// limiters (e.g. control-plane/internal/ratelimit) are similarly simple. +// 10 calls/minute per caller is generous enough that the Network +// Validator's tick-driven retry loop (control-plane/internal/ +// networkvalidator/run.go, ~3s poll interval, retries a failed submit_ +// evidence on the next tick) is not throttled under ordinary transient +// chain-RPC failures, while still tightly bounding one caller's +// worst-case sustained traffic to a provider. +const BANDWIDTH_RATE_LIMIT_WINDOW: Duration = Duration::from_secs(60); +const BANDWIDTH_RATE_LIMIT_MAX_CALLS: u32 = 10; + #[derive(Debug)] pub enum AgentEvent { CmdDeploy { @@ -80,6 +118,134 @@ pub struct AgentGrpcServer { pub identity_manager: Arc, pub inventory_manager: Arc, pub executor: Arc, + pub bandwidth_rate_limiter: BandwidthRateLimiter, +} + +/// Sentinel key for a MeasureBandwidth caller this handler could not +/// identify via its mTLS certificate (peer_certs() returned nothing, or +/// its leaf certificate isn't a parseable Ed25519 certificate). Every +/// unidentifiable caller shares this one bucket, so it still gets rate +/// limited rather than bypassing the limiter entirely -- not a claim that +/// this is any real caller's identity (an all-zero byte string is not a +/// valid Ed25519 public key any real handshake would present). +const UNKNOWN_CALLER: [u8; 32] = [0u8; 32]; + +/// Per-caller rate limiter for MeasureBandwidth (ADR-015 §3), keyed by +/// the caller's raw 32-byte Ed25519 public key extracted from its mTLS +/// leaf certificate (`caller_public_key`) -- the same identity agent-cli's +/// `mtls.rs` allowlist verifier already establishes trust on (ADR-013 +/// §3), reused here purely as a rate-limiting key, not as a second trust +/// decision. A plain fixed window per caller: bounded, simple, and +/// sufficient for this MVP's "cap one caller's worst-case sustained +/// traffic" goal -- see the constants' doc comments for the exact +/// numbers and reasoning. +#[derive(Default)] +pub struct BandwidthRateLimiter { + windows: Mutex>, +} + +impl BandwidthRateLimiter { + pub fn new() -> Self { + Self::default() + } + + /// Returns true and records one call if `key` is still under budget + /// for the current window; false (and records nothing) once the + /// window's budget is exhausted. A stale window (its start is older + /// than BANDWIDTH_RATE_LIMIT_WINDOW) resets rather than accumulating + /// forever. + fn allow(&self, key: [u8; 32]) -> bool { + let mut windows = self + .windows + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let now = Instant::now(); + let entry = windows.entry(key).or_insert((0, now)); + if now.duration_since(entry.1) >= BANDWIDTH_RATE_LIMIT_WINDOW { + *entry = (0, now); + } + if entry.0 >= BANDWIDTH_RATE_LIMIT_MAX_CALLS { + return false; + } + entry.0 += 1; + true + } +} + +/// Extracts the calling client's raw 32-byte Ed25519 public key from the +/// leaf certificate of its mTLS connection, for use as +/// `BandwidthRateLimiter`'s per-caller key. `None` when no peer +/// certificate is available at all -- e.g. this crate exercised outside +/// a real TLS connection (a unit test), or (defense in depth) built +/// without tonic's "tls" feature active in this compilation unit; the +/// caller falls back to the shared `UNKNOWN_CALLER` bucket rather than +/// skipping rate limiting entirely. This does not perform authorization: +/// by the time any Agent RPC handler runs, the mTLS layer (ADR-013 §3) +/// has already decided whether to accept the connection at all. +fn caller_public_key(request: &Request) -> [u8; 32] { + request + .peer_certs() + .and_then(|certs| certs.first().map(|cert| cert.get_ref().to_vec())) + .and_then(|der| extract_ed25519_raw_public_key(&der)) + .unwrap_or(UNKNOWN_CALLER) +} + +/// Parses `der` as an X.509 certificate and returns its raw 32-byte +/// Ed25519 SubjectPublicKeyInfo, or `None` if it isn't a parseable +/// Ed25519 certificate. Byte-for-byte the same approach as agent-cli's +/// `mtls::extract_ed25519_raw_public_key` (duplicated rather than +/// shared -- agent-api cannot depend on agent-cli). +fn extract_ed25519_raw_public_key(der: &[u8]) -> Option<[u8; 32]> { + let (_, certificate) = x509_parser::parse_x509_certificate(der).ok()?; + let spki = certificate.public_key(); + if spki.algorithm.algorithm != x509_parser::oid_registry::OID_SIG_ED25519 { + return None; + } + <[u8; 32]>::try_from(spki.subject_public_key.data.as_ref()).ok() +} + +/// Builds the exact byte sequence signed for a MeasureBandwidth response +/// (ADR-015 §4), deliberately mirroring `solve_challenge`'s existing +/// signing convention -- a domain constant, then fields in a documented, +/// unambiguous order -- rather than inventing a second, differently +/// shaped construction: +/// +/// ```text +/// BANDWIDTH_PROBE_DOMAIN +/// ++ be_u32(len(probe_id)) ++ probe_id +/// ++ upload_payload_hash (32 bytes, fixed width: SHA256 output) +/// ++ be_u32(len(download_payload)) ++ download_payload +/// ++ be_u32(server_processing_ms) (4 bytes, fixed width) +/// ``` +/// +/// `probe_id` and `download_payload` are variable-length and each get an +/// explicit big-endian u32 length prefix (matching how `solve_challenge` +/// frames `challenge_id`); `upload_payload_hash` is always exactly 32 +/// bytes (SHA256's fixed output size) so it needs no length prefix to +/// stay unambiguous; `server_processing_ms` is already a fixed-width u32. +fn bandwidth_signed_bytes( + probe_id: &str, + upload_payload_hash: &[u8], + download_payload: &[u8], + server_processing_ms: u32, +) -> Vec { + let mut signed = Vec::with_capacity( + BANDWIDTH_PROBE_DOMAIN.len() + + 4 + + probe_id.len() + + upload_payload_hash.len() + + 4 + + download_payload.len() + + 4, + ); + signed.extend_from_slice(BANDWIDTH_PROBE_DOMAIN); + signed.extend_from_slice(&(probe_id.len() as u32).to_be_bytes()); + signed.extend_from_slice(probe_id.as_bytes()); + signed.extend_from_slice(upload_payload_hash); + signed.extend_from_slice(&(download_payload.len() as u32).to_be_bytes()); + signed.extend_from_slice(download_payload); + signed.extend_from_slice(&server_processing_ms.to_be_bytes()); + signed } #[tonic::async_trait] @@ -235,6 +401,71 @@ impl provider_agent_service_server::ProviderAgentService for AgentGrpcServer { })) } + async fn measure_bandwidth( + &self, + request: Request, + ) -> Result, Status> { + // Rate-limit before doing any real work, keyed off the caller + // identity extracted from the still-intact request (peer_certs() + // reads connection metadata that into_inner() below discards). + let caller = caller_public_key(&request); + if !self.bandwidth_rate_limiter.allow(caller) { + return Err(Status::resource_exhausted( + "MeasureBandwidth rate limit exceeded for this caller", + )); + } + + let request = request.into_inner(); + if request.probe_id.is_empty() || request.probe_id.len() > MAX_CHALLENGE_ID { + return Err(Status::invalid_argument("probe_id is empty or too long")); + } + if request.upload_payload.len() > MAX_BANDWIDTH_PROBE_BYTES { + return Err(Status::resource_exhausted("upload_payload is too large")); + } + if request.requested_download_bytes as usize > MAX_BANDWIDTH_PROBE_BYTES { + return Err(Status::resource_exhausted( + "requested_download_bytes exceeds the maximum probe size", + )); + } + + // ADR-015 §1: server_processing_ms measures processing only, not + // request deserialization/queueing -- the clock starts here, right + // after into_inner(), and stops just before the response is built. + let started = Instant::now(); + + let mut hasher = Sha256::new(); + hasher.update(&request.upload_payload); + let upload_payload_hash = hasher.finalize().to_vec(); + + // Random, not zeroed: an all-zero download_payload would let a + // lazy/malicious Agent implementation skip real work (and real + // bytes-on-the-wire) while still satisfying a naive length check. + let mut download_payload = vec![0u8; request.requested_download_bytes as usize]; + rand::thread_rng().fill_bytes(&mut download_payload); + + let server_processing_ms = started.elapsed().as_millis().min(u128::from(u32::MAX)) as u32; + + let signed = bandwidth_signed_bytes( + &request.probe_id, + &upload_payload_hash, + &download_payload, + server_processing_ms, + ); + let signature = self + .identity_manager + .sign(&signed) + .await + .map_err(|error| Status::internal(format!("identity signing failed: {error}")))?; + + Ok(Response::new(MeasureBandwidthResponse { + probe_id: request.probe_id, + upload_payload_hash, + download_payload, + server_processing_ms, + signature, + })) + } + async fn stop(&self, request: Request) -> Result, Status> { let req = request.into_inner(); info!( @@ -302,6 +533,8 @@ impl provider_agent_service_server::ProviderAgentService for AgentGrpcServer { #[cfg(test)] mod tests { use super::*; + use agent_core::identity::Ed25519IdentityManager; + use proto::provider_agent_service_server::ProviderAgentService; #[test] fn missing_workload_maps_to_grpc_not_found() { @@ -319,4 +552,311 @@ mod tests { assert_eq!(status.code(), tonic::Code::Internal); } + + struct NoopExecutor; + + #[async_trait] + impl Executor for NoopExecutor { + async fn deploy(&self, _req: DeployRequest) -> anyhow::Result { + unimplemented!("not exercised by MeasureBandwidth tests") + } + async fn stop(&self, _workload_id: &str) -> anyhow::Result<()> { + unimplemented!("not exercised by MeasureBandwidth tests") + } + async fn get_status(&self, _workload_id: &str) -> anyhow::Result { + unimplemented!("not exercised by MeasureBandwidth tests") + } + } + + /// A real generated Ed25519 identity, backed by a temp key file -- + /// exercising the same production `IdentityManager` implementation + /// `solve_challenge`/`measure_bandwidth` sign through, not a fake. + /// The returned `TempDir` must outlive its use (the key file is read + /// once at generation and then held in memory, but keeping the + /// directory alive avoids relying on that implementation detail). + fn test_identity() -> (Arc, tempfile::TempDir) { + let dir = tempfile::tempdir().expect("tempdir"); + let identity = Ed25519IdentityManager::generate(dir.path().join("identity.key")) + .expect("generate identity"); + (Arc::new(identity), dir) + } + + fn test_server(identity_manager: Arc) -> AgentGrpcServer { + let (event_bus, _receiver) = mpsc::channel(1); + AgentGrpcServer { + config: AgentConfig::default(), + event_bus, + identity_manager, + inventory_manager: Arc::new(InventoryManager::new()), + executor: Arc::new(NoopExecutor), + bandwidth_rate_limiter: BandwidthRateLimiter::new(), + } + } + + #[tokio::test] + async fn valid_probe_succeeds_and_hash_and_signature_verify() { + let (identity, _dir) = test_identity(); + let server = test_server(identity.clone()); + + let upload_payload = vec![7u8; 1024]; + let request = Request::new(MeasureBandwidthRequest { + probe_id: "probe-1".to_string(), + upload_payload: upload_payload.clone(), + requested_download_bytes: 512, + }); + + let response = server + .measure_bandwidth(request) + .await + .expect("measure_bandwidth") + .into_inner(); + + let mut hasher = Sha256::new(); + hasher.update(&upload_payload); + assert_eq!(response.upload_payload_hash, hasher.finalize().to_vec()); + assert_eq!(response.download_payload.len(), 512); + assert_eq!(response.probe_id, "probe-1"); + + let signed = bandwidth_signed_bytes( + &response.probe_id, + &response.upload_payload_hash, + &response.download_payload, + response.server_processing_ms, + ); + let public_key_hex = identity.get_public_key().await.expect("public key"); + let public_key = hex::decode(public_key_hex).expect("hex decode public key"); + assert!(identity + .verify(&signed, &response.signature, &public_key) + .await + .expect("verify signature")); + + // A tampered signature must not verify -- confirms the check above + // is actually exercising real verification, not vacuously true. + let mut tampered = response.signature.clone(); + tampered[0] ^= 0xFF; + assert!(!identity + .verify(&signed, &tampered, &public_key) + .await + .expect("verify tampered signature")); + } + + #[tokio::test] + async fn oversized_upload_payload_is_rejected() { + let (identity, _dir) = test_identity(); + let server = test_server(identity); + + let request = Request::new(MeasureBandwidthRequest { + probe_id: "probe-oversized-upload".to_string(), + upload_payload: vec![0u8; MAX_BANDWIDTH_PROBE_BYTES + 1], + requested_download_bytes: 0, + }); + + let status = server + .measure_bandwidth(request) + .await + .expect_err("oversized upload_payload must be rejected"); + assert_eq!(status.code(), tonic::Code::ResourceExhausted); + } + + #[tokio::test] + async fn oversized_requested_download_bytes_is_rejected() { + let (identity, _dir) = test_identity(); + let server = test_server(identity); + + let request = Request::new(MeasureBandwidthRequest { + probe_id: "probe-oversized-download".to_string(), + upload_payload: vec![], + requested_download_bytes: (MAX_BANDWIDTH_PROBE_BYTES + 1) as u32, + }); + + let status = server + .measure_bandwidth(request) + .await + .expect_err("oversized requested_download_bytes must be rejected"); + assert_eq!(status.code(), tonic::Code::ResourceExhausted); + } + + #[tokio::test] + async fn empty_probe_id_is_rejected() { + let (identity, _dir) = test_identity(); + let server = test_server(identity); + + let request = Request::new(MeasureBandwidthRequest { + probe_id: String::new(), + upload_payload: vec![], + requested_download_bytes: 0, + }); + + let status = server + .measure_bandwidth(request) + .await + .expect_err("empty probe_id must be rejected"); + assert_eq!(status.code(), tonic::Code::InvalidArgument); + } + + #[tokio::test] + async fn download_payload_length_exactly_matches_the_request() { + let (identity, _dir) = test_identity(); + let server = test_server(identity); + + let request = Request::new(MeasureBandwidthRequest { + probe_id: "probe-exact-length".to_string(), + upload_payload: vec![], + requested_download_bytes: 12_345, + }); + + let response = server + .measure_bandwidth(request) + .await + .expect("measure_bandwidth") + .into_inner(); + assert_eq!(response.download_payload.len(), 12_345); + } + + /// Two concurrent calls sharing the same probe_id must not corrupt + /// each other's result -- a concurrency-safety check (the handler has + /// no business-logic reason to reject a repeated probe_id; the server + /// deliberately does not enforce probe_id uniqueness). + #[tokio::test] + async fn concurrent_calls_with_the_same_probe_id_do_not_cross_talk() { + let (identity, _dir) = test_identity(); + let server = Arc::new(test_server(identity)); + + let upload_a = vec![1u8; 4096]; + let upload_b = vec![2u8; 8192]; + + let server_a = server.clone(); + let upload_a_clone = upload_a.clone(); + let task_a = tokio::spawn(async move { + let request = Request::new(MeasureBandwidthRequest { + probe_id: "shared-probe-id".to_string(), + upload_payload: upload_a_clone, + requested_download_bytes: 256, + }); + server_a + .measure_bandwidth(request) + .await + .expect("measure_bandwidth a") + .into_inner() + }); + + let server_b = server.clone(); + let upload_b_clone = upload_b.clone(); + let task_b = tokio::spawn(async move { + let request = Request::new(MeasureBandwidthRequest { + probe_id: "shared-probe-id".to_string(), + upload_payload: upload_b_clone, + requested_download_bytes: 512, + }); + server_b + .measure_bandwidth(request) + .await + .expect("measure_bandwidth b") + .into_inner() + }); + + let response_a = task_a.await.expect("task a joined"); + let response_b = task_b.await.expect("task b joined"); + + let mut hasher_a = Sha256::new(); + hasher_a.update(&upload_a); + assert_eq!(response_a.upload_payload_hash, hasher_a.finalize().to_vec()); + assert_eq!(response_a.download_payload.len(), 256); + + let mut hasher_b = Sha256::new(); + hasher_b.update(&upload_b); + assert_eq!(response_b.upload_payload_hash, hasher_b.finalize().to_vec()); + assert_eq!(response_b.download_payload.len(), 512); + } + + #[test] + fn rate_limiter_blocks_a_caller_once_its_window_budget_is_exhausted() { + let limiter = BandwidthRateLimiter::new(); + let caller = [7u8; 32]; + for _ in 0..BANDWIDTH_RATE_LIMIT_MAX_CALLS { + assert!( + limiter.allow(caller), + "expected a call under budget to be allowed" + ); + } + assert!( + !limiter.allow(caller), + "expected a call over budget to be rejected" + ); + // A different caller has its own independent budget. + assert!(limiter.allow([9u8; 32])); + } + + #[tokio::test] + async fn rate_limit_exceeded_is_reported_as_resource_exhausted() { + let (identity, _dir) = test_identity(); + let server = test_server(identity); + + let make_request = || { + Request::new(MeasureBandwidthRequest { + probe_id: "probe-rate-limit".to_string(), + upload_payload: vec![], + requested_download_bytes: 0, + }) + }; + for _ in 0..BANDWIDTH_RATE_LIMIT_MAX_CALLS { + server + .measure_bandwidth(make_request()) + .await + .expect("call under budget must succeed"); + } + let status = server + .measure_bandwidth(make_request()) + .await + .expect_err("call over budget must be rejected"); + assert_eq!(status.code(), tonic::Code::ResourceExhausted); + } + + #[test] + fn caller_public_key_falls_back_to_unknown_when_no_peer_certificate_is_present() { + let request: Request = + Request::new(MeasureBandwidthRequest::default()); + assert_eq!(caller_public_key(&request), UNKNOWN_CALLER); + } + + #[test] + fn extract_ed25519_raw_public_key_matches_a_real_self_signed_certificate() { + use rcgen::{CertificateParams, DnType, KeyPair, PKCS_ED25519}; + + let key_pair = KeyPair::generate_for(&PKCS_ED25519).expect("generate key"); + let raw_key: [u8; 32] = key_pair + .public_key_raw() + .try_into() + .expect("32-byte raw key"); + let mut params = CertificateParams::new(Vec::::new()).expect("params"); + params + .distinguished_name + .push(DnType::CommonName, "bandwidth-probe-test"); + let cert = params.self_signed(&key_pair).expect("self-sign"); + + let extracted = + extract_ed25519_raw_public_key(&cert.der()[..]).expect("parse Ed25519 certificate"); + assert_eq!(extracted, raw_key); + } + + #[test] + fn bandwidth_signed_bytes_changes_when_any_field_changes() { + let base = bandwidth_signed_bytes("probe", &[1u8; 32], b"payload", 42); + assert_ne!( + base, + bandwidth_signed_bytes("other", &[1u8; 32], b"payload", 42) + ); + assert_ne!( + base, + bandwidth_signed_bytes("probe", &[2u8; 32], b"payload", 42) + ); + assert_ne!( + base, + bandwidth_signed_bytes("probe", &[1u8; 32], b"different", 42) + ); + assert_ne!( + base, + bandwidth_signed_bytes("probe", &[1u8; 32], b"payload", 43) + ); + } } diff --git a/provider-agent/crates/agent-cli/src/main.rs b/provider-agent/crates/agent-cli/src/main.rs index da40d11..2b0eaa4 100644 --- a/provider-agent/crates/agent-cli/src/main.rs +++ b/provider-agent/crates/agent-cli/src/main.rs @@ -138,10 +138,19 @@ async fn handle_start(dev: bool) -> Result<()> { identity_manager, inventory_manager, executor, + bandwidth_rate_limiter: agent_api::BandwidthRateLimiter::new(), }; - let router = - tonic::transport::Server::builder().add_service(ProviderAgentServiceServer::new(server)); + // tonic's default per-message limit (4 MiB, both directions) predates + // MeasureBandwidth (ADR-015) and would reject its whole premise -- + // agent_api::MAX_BANDWIDTH_PROBE_BYTES (8 MiB) plus a fixed margin for + // protobuf/gRPC framing overhead beyond the raw payload bytes. + const BANDWIDTH_MESSAGE_SIZE_LIMIT: usize = agent_api::MAX_BANDWIDTH_PROBE_BYTES + 64 * 1024; + let router = tonic::transport::Server::builder().add_service( + ProviderAgentServiceServer::new(server) + .max_decoding_message_size(BANDWIDTH_MESSAGE_SIZE_LIMIT) + .max_encoding_message_size(BANDWIDTH_MESSAGE_SIZE_LIMIT), + ); if dev { if !addr.ip().is_loopback() { anyhow::bail!("plaintext development Agent requires a loopback listen address")