From 95f563cd1f3230a003e7ce9bb943b2794eceed39 Mon Sep 17 00:00:00 2001 From: huangruiteng Date: Sun, 6 Sep 2026 20:55:10 +0800 Subject: [PATCH] fix(reward-memory): guide explicit feedback ingestion from Lark inbox Signed-off-by: huangruiteng --- loopx/capabilities/reward_memory/README.md | 47 +++ .../reward_memory/feedback_hint.py | 107 +++++++ loopx/cli_commands/lark_inbox.py | 19 ++ .../extensions/lark/docs/lark-event-inbox.md | 9 + .../test_reward_memory_feedback_hint.py | 283 ++++++++++++++++++ 5 files changed, 465 insertions(+) create mode 100644 loopx/capabilities/reward_memory/feedback_hint.py create mode 100644 tests/capabilities/test_reward_memory_feedback_hint.py diff --git a/loopx/capabilities/reward_memory/README.md b/loopx/capabilities/reward_memory/README.md index cd9403a4cb..42086dcf5f 100644 --- a/loopx/capabilities/reward_memory/README.md +++ b/loopx/capabilities/reward_memory/README.md @@ -144,6 +144,53 @@ verified. Issue Fix continues normally when the experiment is disabled, unavailable, rejected by guards, or fails exact readback. Invalid or non-v1 configuration resolves unavailable with both automatic flags false. +### Inbox feedback review (explicit ingestion) + +When a registry-routed `loopx lark-inbox drain --goal-id --agent-id +` returns messages, it also emits `reward_memory_feedback_review` if +Reward Memory is enabled for that agent and has an active, writable +`scoped_feedback` route with an enabled standing policy and exact +`peer_ref=agent:`. The hint lists configured destinations and a preview +command bound to the same registry, Goal and Agent. It is advisory: there is no +provider call, automatic candidate creation, new permission, or ACK gate. +Both JSON and Markdown drains expose it. Empty/disabled inboxes, disabled or +invalid memory configurations, unconfigured agents, incompatible routes, and +explicit `--config`/`--project` overrides retain their previous output. + +The agent reviews the conversation before choosing what, if anything, to learn: + +1. Verify the source actor and existing authority, current evidence, conflicts + and applicability. A policy's allowed roles are not proof of a sender's role. + A disagreement is not a universal ban, and one-off task state belongs in + Todo/vision rather than durable preference memory. +2. Distill only confirmed reusable feedback into an applicable configured + `soft_preference`, `procedural_experience`, or independently authorized + `hard_policy` route. Never widen the route or enable automation to make an + event pass. Keep raw chat and credentials out of the event. +3. Prepare `{adapter, event, observed_at}` using the + [scoped feedback fixture](../../../examples/fixtures/reward-memory-scoped-feedback-ingest.public.json) + for field shape only. The event uses + `schema_version=scoped_feedback_reward_memory_event_v0`, a stable + `feedback_ref`, actual `source`, `reasoning`, `guard_context`, compact + `content_summary`, `target_class`, and exact identity/surface/revision/action + scope. Advisory classes require empty `requested_action_scopes`; allowed + policy scopes do not grant advisory memory action authority. Do not copy the + fixture's actor or verified-guard assertions. +4. Replace the hint's input placeholder and preview `ingest-event` without + `--execute`. Inspect its guards. Only then execute within the standing + policy and verify `exact_readback_verified` and + `memory_available_for_recall`; a preview is not a learned memory. +5. Finish normal reply/material-review/ACK. No reusable feedback or unavailable + memory is an honest no-memory outcome, not a reason to block the inbox or + repeatedly ask the user for permission. Actual write failures remain visible + in the existing ingestion receipt. + +`automatic_ingest=false` does **not** prohibit this explicit workflow. Enabling +it also does **not** wire raw inbox messages into ingestion. Issue Fix's compact +feedback adapter and recall hooks share the same core but are not a general +inbox-to-memory feedback loop. Disable the hint by disabling Reward Memory for +the agent (or its applicable route); no extra store, queue or scheduler exists. + ## Five first-class classes | Class | Source and scope | Authority and use | Lifecycle | diff --git a/loopx/capabilities/reward_memory/feedback_hint.py b/loopx/capabilities/reward_memory/feedback_hint.py new file mode 100644 index 0000000000..aba6d346ad --- /dev/null +++ b/loopx/capabilities/reward_memory/feedback_hint.py @@ -0,0 +1,107 @@ +"""Effect-free guidance for agents reviewing an inbox's captured feedback.""" + +from __future__ import annotations + +import shlex +from pathlib import Path +from typing import Any + +from .experiment import ( + resolve_reward_memory_experiment, + resolve_reward_memory_surface_config, +) +from .scoped_feedback import SCOPED_FEEDBACK_ADAPTER + + +def build_feedback_review_hint( + *, registry_path: Path, goal_id: str | None, agent_id: str | None +) -> dict[str, Any] | None: + """Describe explicit ingestion, never inspect messages or call a provider. + + The caller must first establish an enabled, registry-routed inbox with + returned items. Automation flags govern hooks, not this explicit path. + """ + if not goal_id or not agent_id: + return None + try: + status, config = resolve_reward_memory_experiment( + registry_path=registry_path, goal_id=goal_id, agent_id=agent_id + ) + except (OSError, ValueError): + return None + if config is None: + return None + routes = [] + for surface_id, surface in sorted(config["surfaces"].items()): + if surface["adapter"] != SCOPED_FEEDBACK_ADAPTER: + continue + route = resolve_reward_memory_surface_config(config, surface_id) + corpus, policy = route["corpus"], route["standing_policy"] + scope = corpus["scope"] + if ( + not policy["enabled"] + or scope.get("peer_ref") != f"agent:{agent_id}" + or corpus["lifecycle"]["state"] != "active" + or corpus["write_authority"] in {"read_only", "ephemeral_runtime"} + ): + continue + routes.append( + { + "surface_id": surface_id, + "corpus_id": corpus["corpus_id"], + "target_class": corpus["class_id"], + "scope": scope | {"surface_ids": [surface_id]}, + "allowed_source_kinds": policy["allowed_source_kinds"], + "allowed_actor_roles": policy["allowed_actor_roles"], + "allowed_action_scopes": policy["allowed_action_scopes"], + } + ) + if not routes: + return None + preview = [ + "loopx", + "--format", + "json", + "--registry", + str(registry_path.expanduser()), + "reward-memory", + "ingest-event", + "--goal-id", + goal_id, + "--agent-id", + agent_id, + "--input", + "", + ] + return { + "schema_version": "reward_memory_feedback_review_hint_v0", + "advisory_only": True, + "automatic_ingest": status["automatic_ingest"], + "automatic_ingest_required": False, + "grants_new_action_authority": False, + "blocks_inbox_settlement": False, + "provider_calls_performed": False, + "routes": routes, + "preview_command": shlex.join(preview), + "instruction": ( + "While triaging these messages, consider confirmed, reusable feedback for " + "Reward Memory. Distill a compact scoped lesson, not raw chat; verify the " + "source actor, authority, freshness, current artifact and conflicts. " + "Choose only an applicable configured route below; allowed actor roles " + "are constraints, not proof of the sender's authority. Do not turn " + "disagreement or a one-off opinion into a universal prohibition. " + "Use soft preferences or procedural experience where applicable; hard " + "policy still requires independently verified existing authority. " + "For advisory classes, requested_action_scopes must be empty. " + "Prepare {adapter: scoped_feedback, event: scoped_feedback_reward_memory_event_v0, " + "observed_at} using the documented event fields and a stable feedback_ref; " + "replace the input placeholder and preview before adding --execute. " + "Do not copy fixture actor/guard assertions or expand scope to pass guards. " + "Automatic ingest being off does not disable explicit ingest-event. " + "After an authorized write, inspect the ingest receipt and exact readback; " + "do not claim learning from a preview or failed write. If no reusable " + "lesson exists, evidence conflicts, or memory is unavailable, continue " + "normal reply/material-review/ACK with an honest rationale; no new user gate." + ), + "event_reference": "loopx/capabilities/reward_memory/README.md", + } diff --git a/loopx/cli_commands/lark_inbox.py b/loopx/cli_commands/lark_inbox.py index ede4324a39..896f4133d3 100644 --- a/loopx/cli_commands/lark_inbox.py +++ b/loopx/cli_commands/lark_inbox.py @@ -8,6 +8,7 @@ from pathlib import Path from ..capabilities.issue_fix.provider_hooks import IssueFixReviewerProviderHooks +from ..capabilities.reward_memory.feedback_hint import build_feedback_review_hint from ..capabilities.reward_memory.outbound import outbound_guidance_hook from ..control_plane.capability_hooks import ( TURN_START_HOOK_RESULT_SCHEMA_VERSION, @@ -555,6 +556,11 @@ def _render(payload: dict[str, object]) -> str: + str(guidance.get("review_digest")) + ". This is not a request for user approval." ) + feedback = payload.get("reward_memory_feedback_review") + if isinstance(feedback, dict): + lines.append(f"- Reward Memory (advisory): {feedback['instruction']}") + lines.append(f"- preview: {feedback['preview_command']}") + lines.append("- configured routes: " + json.dumps(feedback["routes"])) return "\n".join(lines).rstrip() + "\n" @@ -602,6 +608,19 @@ def handle_lark_inbox_command( config_path=config_path, limit=args.limit, ) + if ( + payload.get("enabled") is True + and payload.get("items") + and not getattr(args, "config", None) + and not getattr(args, "project", None) + ): + hint = build_feedback_review_hint( + registry_path=registry_path, + goal_id=getattr(args, "goal_id", None), + agent_id=getattr(args, "agent_id", None), + ) + if hint is not None: + payload["reward_memory_feedback_review"] = hint elif args.lark_inbox_command == "ack": payload = acknowledge_routed_lark_event_inbox( project=project, diff --git a/loopx/extensions/lark/docs/lark-event-inbox.md b/loopx/extensions/lark/docs/lark-event-inbox.md index a496d85a90..31329a1ff4 100644 --- a/loopx/extensions/lark/docs/lark-event-inbox.md +++ b/loopx/extensions/lark/docs/lark-event-inbox.md @@ -31,6 +31,15 @@ parent sender is the app id of the configured profile. A reply to a person, another app, or an unverifiable parent remains captured but does not wake the agent. The agent does not need to keep a websocket open. +When Lark inbox and the same registered agent's Reward Memory are both enabled, +a non-empty registry-routed drain can also return an advisory +`reward_memory_feedback_review` hint. It asks the agent to review reusable +feedback and preview the existing scoped `reward-memory ingest-event` command; +it does not ingest chat, grant authority, or change settlement/ACK requirements. +This explicit path does not require `automatic_ingest=true`. See the +[Reward Memory inbox workflow](../../../capabilities/reward_memory/README.md#inbox-feedback-review-explicit-ingestion) +for eligibility, source verification, write/readback and default-off behavior. + ### Optional turn-start Agent reading hook Realtime collection is the preferred ingress, but a long-running Agent may also diff --git a/tests/capabilities/test_reward_memory_feedback_hint.py b/tests/capabilities/test_reward_memory_feedback_hint.py new file mode 100644 index 0000000000..772cdadf4d --- /dev/null +++ b/tests/capabilities/test_reward_memory_feedback_hint.py @@ -0,0 +1,283 @@ +from __future__ import annotations + +import json +import shlex + +import pytest + +from loopx.capabilities.reward_memory.feedback_hint import build_feedback_review_hint +from loopx.cli import main +from loopx.cli_commands import lark_inbox +from tests.capabilities.test_agent_turn_recall import raw_config +from tests.extensions.test_lark_inbox_reactions import _fixture + + +def experiment(tmp_path): + project = tmp_path / "project" + config = project / ".loopx/config/reward-memory/experiment.json" + config.parent.mkdir(parents=True, exist_ok=True) + config.write_text(json.dumps(raw_config())) + registry = tmp_path / "registry.json" + registry.write_text( + json.dumps( + { + "schema_version": 1, + "goals": [ + { + "id": "reward-memory-goal", + "repo": str(project), + "coordination": {"registered_agents": ["pilot", "meta"]}, + "control_plane": { + "reward_memory": { + "enabled": True, + "experimental": True, + "enabled_agents": ["pilot"], + "config_path": ".loopx/config/reward-memory/experiment.json", + } + }, + } + ], + } + ) + ) + return registry, project, config + + +def hint(registry, **overrides): + return build_feedback_review_hint( + **{ + "registry_path": registry, + "goal_id": "reward-memory-goal", + "agent_id": "pilot", + **overrides, + } + ) + + +@pytest.mark.parametrize("automatic", [False, True]) +def test_hint_reuses_explicit_route_without_automation_or_provider( + tmp_path, monkeypatch, automatic +): + registry, _, config = experiment(tmp_path) + raw = raw_config() + raw["automation"] = { + "automatic_recall": automatic, + "automatic_ingest": automatic, + "fail_open": True, + } + config.write_text(json.dumps(raw)) + + def forbidden(*a, **kw): + pytest.fail("a hint must not construct a memory provider") + + monkeypatch.setattr( + "loopx.capabilities.reward_memory.application.build_context_provider", forbidden + ) + result = hint(registry) + assert result["automatic_ingest"] is automatic + assert not result["automatic_ingest_required"] + assert result["advisory_only"] + assert not result["grants_new_action_authority"] + assert not result["blocks_inbox_settlement"] + assert not result["provider_calls_performed"] + assert result["routes"][0]["target_class"] == "soft_preference" + assert result["routes"][0]["scope"]["peer_ref"] == "agent:pilot" + argv = shlex.split(result["preview_command"]) + assert argv == [ + "loopx", + "--format", + "json", + "--registry", + str(registry), + "reward-memory", + "ingest-event", + "--goal-id", + "reward-memory-goal", + "--agent-id", + "pilot", + "--input", + "", + ] + assert ( + "authority" in result["instruction"] + and "exact readback" in result["instruction"] + ) + assert "viking://" not in json.dumps(result) + + +@pytest.mark.parametrize( + "condition", + [ + "disabled", + "other_agent", + "unregistered", + "missing_agent", + "missing_goal", + "invalid_config", + "wrong_peer", + "unsupported_adapter", + "policy_disabled", + "read_only", + "missing_registry", + ], +) +def test_ineligible_routes_produce_no_hint(tmp_path, condition): + registry, _, config = experiment(tmp_path) + overrides = {} + raw = raw_config() + if condition == "disabled": + payload = json.loads(registry.read_text()) + payload["goals"][0]["control_plane"]["reward_memory"]["enabled"] = False + registry.write_text(json.dumps(payload)) + elif condition in {"other_agent", "unregistered", "missing_agent"}: + overrides["agent_id"] = { + "other_agent": "meta", + "unregistered": "stranger", + "missing_agent": None, + }[condition] + elif condition == "missing_goal": + overrides["goal_id"] = None + elif condition == "missing_registry": + registry = tmp_path / "absent.json" + elif condition == "invalid_config": + raw = {} + elif condition == "wrong_peer": + raw = raw_config(peer_ref="agent:meta") + elif condition == "unsupported_adapter": + raw["surfaces"][0]["adapter"] = "issue_fix_maintainer_feedback" + elif condition == "policy_disabled": + raw["corpora"][0]["standing_policy"]["enabled"] = False + elif condition == "read_only": + raw["corpora"][0]["corpus"]["write_authority"] = "read_only" + config.write_text(json.dumps(raw)) + assert hint(registry, **overrides) is None + + +def test_generated_command_previews_reviewed_event_and_rejects_scope_expansion( + tmp_path, capsys +): + registry, _, _ = experiment(tmp_path) + result = hint(registry) + route = result["routes"][0] + event = { + **{k: v for k, v in route["scope"].items() if k != "surface_ids"}, + "schema_version": "scoped_feedback_reward_memory_event_v0", + "feedback_ref": "feedback:example:summary", + "surface_id": route["surface_id"], + "revision_ref": "revision:example", + "target_class": "soft_preference", + "content_summary": "Use a concise summary followed by complete handoff details.", + "source": { + "source_kind": "explicit_user_instruction", + "source_ref": "feedback:example:summary", + "actor_ref": "user:example", + "actor_role": "verified_project_owner_or_operator", + }, + "reasoning": { + "summary": "Confirmed scoped formatting feedback.", + "confidence": "high", + }, + "guard_context": { + "source_freshness": "current", + "conflict_state": "clear", + "current_artifact_verified": True, + }, + "requested_action_scopes": [], + "raw_content_captured": False, + } + path = tmp_path / "compact-event.json" + command = shlex.split(result["preview_command"])[1:] + command[-1] = str(path) + for peer, expected in [("agent:pilot", "planned"), ("agent:meta", "guard_blocked")]: + event["peer_ref"] = peer + path.write_text( + json.dumps( + { + "adapter": "scoped_feedback", + "event": event, + "observed_at": "2026-08-01T00:00:00Z", + } + ) + ) + assert main(command) == 0 + receipt = json.loads(capsys.readouterr().out) + assert receipt["status"] == expected, receipt["guard"] + assert receipt["experiment"]["automatic_ingest"] is False + assert not receipt["external_writes_performed"] + assert not receipt["exact_readback_verified"] + + +@pytest.mark.parametrize( + "condition", + [ + "enabled", + "empty", + "disabled", + "explicit_config", + "project_override", + "memory_disabled", + ], +) +def test_real_drain_entrypoint_preserves_inbox_and_disabled_parity( + tmp_path, monkeypatch, capsys, condition +): + inbox_config, _, inbox_project = _fixture(tmp_path / "project", lifecycle=False) + registry, project, _ = experiment(tmp_path) + # Bind the synthetic project's existing local inbox through its registry. + payload = json.loads(registry.read_text()) + payload["goals"][0]["control_plane"]["lark_event_inbox"] = { + "enabled": True, + "config_path": str(inbox_config), + } + registry.write_text(json.dumps(payload)) + monkeypatch.setattr(lark_inbox, "_resolve_lark_activation", lambda *a, **kw: {}) + original = lark_inbox.inspect_routed_lark_event_inbox( + project=inbox_project, + config_path=inbox_config, + limit=20, + ) + if condition in {"empty", "disabled"}: + observed = dict(original) + if condition == "empty": + observed["items"] = [] + else: + observed["enabled"] = False + monkeypatch.setattr( + lark_inbox, "inspect_routed_lark_event_inbox", lambda **kw: dict(observed) + ) + original = observed + extra = [] + if condition == "explicit_config": + extra = ["--config", str(inbox_config), "--project", str(project)] + if condition == "project_override": + extra = ["--project", str(project)] + if condition == "memory_disabled": + payload["goals"][0]["control_plane"]["reward_memory"]["enabled"] = False + registry.write_text(json.dumps(payload)) + assert ( + main( + [ + "--format", + "json", + "--registry", + str(registry), + "lark-inbox", + "drain", + "--goal-id", + "reward-memory-goal", + "--agent-id", + "pilot", + *extra, + ] + ) + == 0 + ) + result = json.loads(capsys.readouterr().out) + review = result.pop("reward_memory_feedback_review", None) + result.pop("extension_activation", None) + assert result == original + assert bool(review) is (condition == "enabled") + if review: + rendered = lark_inbox._render({"reward_memory_feedback_review": review}) + assert review["preview_command"] in rendered + assert "advisory" in rendered