diff --git a/AgentMemorySystem.py b/AgentMemorySystem.py index 6d0e5aa..8371682 100644 --- a/AgentMemorySystem.py +++ b/AgentMemorySystem.py @@ -1,365 +1,5 @@ -from scheme_b_v330 import * -import scheme_b_v330 as v330 +from scheme_b_v336 import * # noqa: F401,F403 +import scheme_b_v336 as v336 # noqa: F401 -from dataclasses import dataclass -from typing import Dict, Optional - -import torch -import torch.nn as nn -import torch.nn.functional as F - -_dev = v330._dev -_Node = v330._Node - - -def _resolve_dtype(name: str): - return {"bf16": torch.bfloat16, "fp16": torch.float16, "fp32": torch.float32}[name] - - -@dataclass -class Cfg(v330.Cfg): - llm_name: str = "Qwen/Qwen2.5-1.5B-Instruct" - llm_dtype: str = "bf16" - use_chat_template_for_gen: bool = False - d_LLM: int = 1536 - vocab_size: int = 151936 - degen_early_punct_penalty: float = 6.0 - degen_early_newline_penalty: float = 6.0 - content_bias_scale: float = 4.0 - cfg_scale: float = 2.0 - tail_head_hidden: int = 1024 - late_newline_penalty: float = 20.0 - newline_hard_gate_min_step: int = 12 - newline_hard_gate_min_content: int = 6 - eos_hard_mask_steps: int = 10 - wte_neighbor_max_vocab: int = 60000 - - def __post_init__(self): - super().__post_init__() - assert self.llm_dtype in ("bf16", "fp16", "fp32") - - -class LLMBackbone(nn.Module): - def __init__(self, name: str, dtype_name: str = "bf16"): - super().__init__() - from transformers import AutoModelForCausalLM, AutoTokenizer - - self.name = name - self._dtype = _resolve_dtype(dtype_name) - self.tokenizer = AutoTokenizer.from_pretrained(name, trust_remote_code=True) - if self.tokenizer.pad_token is None: - if self.tokenizer.eos_token is not None: - self.tokenizer.pad_token = self.tokenizer.eos_token - else: - raise ValueError(f"Tokenizer for {name} has no pad/eos token") - self.model = AutoModelForCausalLM.from_pretrained( - name, - torch_dtype=self._dtype, - trust_remote_code=True, - ) - for p in self.model.parameters(): - p.requires_grad_(False) - self.model.eval() - cfg = self.model.config - self.d_model = cfg.hidden_size - self.vocab_size = cfg.vocab_size - self.n_layers = cfg.num_hidden_layers - self.has_chat_template = getattr(self.tokenizer, "chat_template", None) is not None - with torch.no_grad(): - self._wte_fp32 = self.model.get_input_embeddings().weight.detach().float().clone() - - def input_embedding_weight(self) -> torch.Tensor: - return self._wte_fp32 - - def embed_tokens(self, ids: torch.Tensor) -> torch.Tensor: - return self.model.get_input_embeddings()(ids) - - def to(self, *args, **kwargs): - super().to(*args, **kwargs) - return self - - def forward(self, ids: torch.Tensor, attention_mask: torch.Tensor, prefix: Optional[torch.Tensor] = None): - te = self.embed_tokens(ids) - if prefix is not None: - prefix_cast = prefix.to(te.dtype) - inputs_embeds = torch.cat([prefix_cast, te], dim=1) - B, P = prefix_cast.shape[:2] - pm = torch.ones(B, P, device=ids.device, dtype=attention_mask.dtype) - ext_mask = torch.cat([pm, attention_mask], dim=1) - pl = P - else: - inputs_embeds = te - ext_mask = attention_mask - pl = 0 - out = self.model( - inputs_embeds=inputs_embeds, - attention_mask=ext_mask, - output_hidden_states=True, - use_cache=False, - return_dict=True, - ) - return { - "logits": out.logits.float(), - "hs": [h.float() for h in out.hidden_states], - "pl": pl, - "mask": ext_mask, - } - - def build_chat_text(self, user_text: str) -> str: - if not self.has_chat_template: - return user_text - msgs = [{"role": "user", "content": user_text}] - return self.tokenizer.apply_chat_template(msgs, tokenize=False, add_generation_prompt=True) - - -class FiberConnection(v330.FiberConnection): - def __init__(self, d_M, d_F, metric, grad_coupling=True): - super().__init__(d_M, d_F, metric, grad_coupling=grad_coupling) - idx = torch.triu_indices(d_M, d_M) - self.register_buffer('_tri_r', idx[0], persistent=False) - self.register_buffer('_tri_c', idx[1], persistent=False) - - def forward(self, x, v): - g = self.metric(x) - gf = g[:, self._tri_r, self._tri_c] - if not self.grad_coupling: - gf = gf.detach() - raw = self.net(torch.cat([x, v, gf], -1)).reshape(-1, self.d_F, self.d_F) - return (raw - raw.transpose(1, 2)) / 2 - - -class AMM(v330.AMM): - def __init__(self, c): - super().__init__(c) - self.conn = FiberConnection(c.d_M, c.d_F, self.metric, grad_coupling=True) - self.trans = v330.FiberTransporter(self.conn, c) - - -class MemLLM(v330.MemLLM): - def __init__(self, c): - super().__init__(c) - self.amm = AMM(c) - self.backbone = None - - def load(self, name: Optional[str] = None, dtype_name: Optional[str] = None): - name = name or self.c.llm_name - dev = next(self.parameters()).device - dtype_name = dtype_name or self.c.llm_dtype - if dev.type == 'mps' and dtype_name == 'bf16': - dtype_name = 'fp16' - self.backbone = LLMBackbone(name, dtype_name=dtype_name) - self.backbone.to(dev) - self.tok = self.backbone.tokenizer - self.c.d_LLM = self.backbone.d_model - self.c.vocab_size = self.backbone.vocab_size - if self.amm.ctx.f1.in_features != self.c.d_LLM: - self.amm = AMM(self.c).to(dev) - self.bridge = v330.EmbBridge(self.c).to(dev) - self.semantic_probe = v330.PrefixSemanticProbe(self.c.d_LLM, self.c.L_mem, self.c.d_F).to(dev) - self.vocab_proj = v330.MemoryVocabProjector(self.c.d_F, self.c.d_LLM).to(dev) - self.layer_pool = v330.AdaptiveLayerPool(self.backbone.n_layers + 1, self.c.d_LLM).to(dev) - self.content_classifier = v330.ContentTokenClassifier(self.tok, self.c) - self._degen_guard = v330.DegenerationGuard(self.tok, self.c, self.content_classifier) - wte_fp32 = self.backbone.input_embedding_weight() - with torch.no_grad(): - si = min(5000, wte_fp32.shape[0]) - idx = torch.randperm(wte_fp32.shape[0])[:si] - self.bridge.aligner._target_std.fill_(float(wte_fp32[idx].std().item())) - self.bridge.aligner._calibrated = True - self._wte_normed = F.normalize(wte_fp32.detach().cpu(), dim=-1, eps=1e-8) - self.amm.wte_normed = self._wte_normed - self._build_wte_neighbor_cache() - self._compute_filler_centroid() - return self - - def _build_wte_neighbor_cache(self): - if self.backbone is None or self.content_classifier is None: - return - V = self.backbone.vocab_size - if V > self.c.wte_neighbor_max_vocab: - self._wte_neighbor_cache = {} - return - wte_n = self._wte_normed - cc = self.content_classifier - valid = [t for t in sorted(cc.content_ids) if t < wte_n.shape[0]] - self._wte_neighbor_cache = {} - K = self.c.wte_neighbor_k - thresh = self.c.wte_neighbor_threshold - batch_size = 500 - for start in range(0, len(valid), batch_size): - batch_ids = valid[start : start + batch_size] - batch_t = torch.tensor(batch_ids, device=wte_n.device) - sims = wte_n[batch_t] @ wte_n.T - topk_vals, topk_ids = sims.topk(K + 1, dim=-1) - for i, tid in enumerate(batch_ids): - neighbors = [] - for score, nid in zip(topk_vals[i], topk_ids[i]): - nid_int = int(nid.item()) - if nid_int == tid: - continue - if score.item() >= thresh and nid_int in cc.content_ids: - neighbors.append(nid_int) - self._wte_neighbor_cache[tid] = neighbors - - def _expand_content_ids(self, content_ids): - if not self._wte_neighbor_cache: - return content_ids - expanded = set(content_ids) - for tid in content_ids: - expanded.update(self._wte_neighbor_cache.get(tid, [])) - return list(expanded) - - def _compute_filler_centroid(self): - if self.content_classifier is None or self.backbone is None: - self._filler_centroid = None - return - wte = self.backbone.input_embedding_weight() - valid = [tid for tid in sorted(self.content_classifier.filler_ids) if tid < wte.shape[0]] - if len(valid) < 3: - self._filler_centroid = None - return - filler_vecs = wte[torch.tensor(valid)] - centroid = F.normalize(filler_vecs.mean(0), dim=-1, eps=1e-8) - self._filler_centroid = centroid.to(next(self.parameters()).device) - - def fwd(self, ids, mask, prefix=None): - return self.backbone(ids, mask, prefix=prefix) - - - def forward_logits_only(self, ids, attention_mask, prefix=None): - te = self.backbone.embed_tokens(ids) - if prefix is not None: - prefix_cast = prefix.to(te.dtype) - inputs_embeds = torch.cat([prefix_cast, te], dim=1) - B, P = prefix_cast.shape[:2] - pm = torch.ones(B, P, device=ids.device, dtype=attention_mask.dtype) - ext_mask = torch.cat([pm, attention_mask], dim=1) - pl = P - else: - inputs_embeds = te - ext_mask = attention_mask - pl = 0 - out = self.backbone.model( - inputs_embeds=inputs_embeds, - attention_mask=ext_mask, - output_hidden_states=False, - use_cache=False, - return_dict=True, - ) - return {"logits": out.logits.float(), "pl": pl, "mask": ext_mask} - - def _compute_vocab_bias(self, fiber_summary): - if fiber_summary is None: - return None - with torch.no_grad(): - mem_emb = self.vocab_proj.proj(fiber_summary).float().cpu() - mem_n = F.normalize(mem_emb, dim=-1, eps=1e-8) - wte_n = self._wte_normed - parts = [] - chunk = 8192 - for start in range(0, wte_n.shape[0], chunk): - parts.append(mem_n @ wte_n[start : start + chunk].T) - return torch.cat(parts, dim=-1).to(fiber_summary.device) - - -class Trainer(v330.Trainer): - def encoder_throughput_loss(self, ids, mask, fiber): - B = ids.shape[0] - dev = ids.device - fiber_unsq = fiber.unsqueeze(1) - mem_mask_ones = torch.ones(B, 1, device=dev) - prefix = self.m.bridge.inject(fiber_unsq, mem_mask_ones, fiber_summary=fiber) - o2 = self.m.forward_logits_only(ids, mask, prefix) - lg = o2['logits'][:, o2['pl']:-1] - tg = ids[:, 1:] - ml = min(lg.shape[1], tg.shape[1]) - if ml == 0: - return torch.tensor(0.0, device=dev, requires_grad=True) - return F.cross_entropy(lg[:, :ml].reshape(-1, lg.shape[-1]), tg[:, :ml].reshape(-1)) - - def _recon_forward(self, text): - tk = self.m.tok(text, return_tensors='pt', padding=True, truncation=True) - dev = next(self.m.parameters()).device - ids, mask = tk['input_ids'].to(dev), tk['attention_mask'].to(dev) - with torch.no_grad(): - bo = self.m.fwd(ids, mask) - prefix = self.m._get_prefix(bo['hs'], mask, update_stats=False, ids=ids) - o = self.m.forward_logits_only(ids, mask, prefix) - lg = o['logits'][:, o['pl']:-1] - tg = ids[:, 1:] - ml = min(lg.shape[1], tg.shape[1]) - if ml == 0: - zero = ids.new_tensor(0.0, dtype=torch.float, requires_grad=True) - return zero, prefix, self.m.bridge._last_fiber_summary - l_r = F.cross_entropy(lg[:, :ml].reshape(-1, lg.shape[-1]), tg[:, :ml].reshape(-1)) - fs = self.m.bridge._last_fiber_summary - if fs is None: - fs = torch.zeros(1, self.c.d_F, device=dev) - return l_r, prefix, fs - - def semantic_alignment_loss(self, fiber, target_ids, target_mask): - dev = fiber.device - wte = self.m.backbone.input_embedding_weight().to(dev) - vocab_logits = self.m.vocab_proj(fiber, wte) - B, V = vocab_logits.shape - cc = self.m.content_classifier - if cc is None: - return torch.tensor(0.0, device=dev, requires_grad=True) - target = torch.zeros(B, V, device=dev) - valid_count = 0 - for b in range(B): - valid = target_ids[b][target_mask[b].bool()].tolist() - content_ids = cc.get_content_ids_from_tokens(valid) - if content_ids: - uids = [uid for uid in set(content_ids) if uid < V] - if uids: - target[b, uids] = 1.0 / len(uids) - valid_count += 1 - if valid_count == 0: - return torch.tensor(0.0, device=dev, requires_grad=True) - log_probs = F.log_softmax(vocab_logits / self.c.semantic_align_temp, dim=-1) - return F.kl_div(log_probs, target, reduction="none").sum(-1).mean() - - def vocab_anchor_loss(self, prefix): - dev = prefix.device - wte = self.m.backbone.input_embedding_weight().to(dev) - pn = F.normalize(prefix.reshape(-1, prefix.shape[-1]), dim=-1) - wn = F.normalize(wte, dim=-1) - sim = pn @ wn.T - return -sim.topk(self.c.vocab_anchor_topk, dim=-1).values.mean() - - def tail_semantic_anchor_loss(self, fiber, ids, mask): - if not (self.c.use_content_semantic_tail and self.c.content_tail_slots > 0): - return torch.tensor(0.0, device=fiber.device, requires_grad=True) - tail = self.m.bridge.tail_head(fiber) - if tail is None: - return torch.tensor(0.0, device=fiber.device, requires_grad=True) - dev = fiber.device - wte = self.m.backbone.input_embedding_weight().to(dev) - B, _, _ = tail.shape - V = wte.shape[0] - cc = self.m.content_classifier - if cc is None: - return torch.tensor(0.0, device=dev, requires_grad=True) - losses = [] - tn = F.normalize(tail, dim=-1) - wn = F.normalize(wte, dim=-1) - for b in range(B): - valid = ids[b][mask[b].bool()].tolist() - content_tids = [t for t in set(cc.get_content_ids_from_tokens(valid)) if t < V] - if not content_tids: - continue - target = torch.zeros(V, device=dev) - target[content_tids] = 1.0 / len(content_tids) - slot_logits = tn[b] @ wn.T / 0.3 - log_probs = F.log_softmax(slot_logits, dim=-1) - losses.append( - F.kl_div( - log_probs, - target.unsqueeze(0).expand_as(log_probs), - reduction="none", - ).sum(-1).mean() - ) - if not losses: - return torch.tensor(0.0, device=dev, requires_grad=True) - return torch.stack(losses).mean() +_Node = v336._Node +_dev = v336._dev diff --git a/reports/v336_blackbox/report.json b/reports/v336_blackbox/report.json new file mode 100644 index 0000000..ed0991a --- /dev/null +++ b/reports/v336_blackbox/report.json @@ -0,0 +1,3600 @@ +{ + "generated_at_epoch": 1776608258.0608523, + "elapsed_seconds": 1169.4885413646698, + "checks": [ + { + "name": "leaf_capacity_stability", + "passed": true, + "detail": "{\"per_seed\": [{\"seed\": 0, \"depth\": 6, \"count\": 240, \"violations\": [], \"consistency\": [], \"passed\": true}, {\"seed\": 1, \"depth\": 6, \"count\": 240, \"violations\": [], \"consistency\": [], \"passed\": true}, {\"seed\": 2, \"depth\": 6, \"count\": 240, \"violations\": [], \"consistency\": [], \"passed\": true}, {\"seed\": 3, \"depth\": 6, \"count\": 240, \"violations\": [], \"consistency\": [], \"passed\": true}, {\"seed\": 4, \"depth\": 6, \"count\": 240, \"violations\": [], \"consistency\": [], \"passed\": true}, {\"seed\": 5, \"depth\": 5, \"count\": 240, \"violations\": [], \"consistency\": [], \"passed\": true}, {\"seed\": 6, \"depth\": 6, \"count\": 240, \"violations\": [], \"consistency\": [], \"passed\": true}, {\"seed\": 7, \"depth\": 5, \"count\": 240, \"violations\": [], \"consistency\": [], \"passed\": true}]}" + }, + { + "name": "degenerate_direction_boundary", + "passed": true, + "detail": "{\"depth\": 47, \"count\": 100, \"violations\": [], \"consistency\": [], \"seed\": 17}" + }, + { + "name": "metric_trainability", + "passed": true, + "detail": "{\"training_info\": {\"total\": 427.3717041015625, \"recon\": 2.9565038681030273, \"contrast\": 17888.765625, \"holonomy\": 5206.763671875, \"write_policy\": 1.2801257371902466, \"semantic_probe\": 0.0, \"dir_diversity\": 0.0, \"reranker_ranking\": 0.0, \"encoder_throughput\": 3.7922558784484863, \"vocab_anchor\": -0.0, \"semantic_alignment\": 9.940794944763184, \"tail_semantic_anchor\": 9.934552192687988, \"grad_norms\": {\"ctx_encoder\": 5.512282921135631e-12, \"fib_encoder\": 2.2757680619031593e-09, \"dir_predictor\": 0.0, \"fiber_connection\": 4.7619314000630244e-08, \"fiber_attn\": 5.288609216022044e-11, \"reranker\": 9.430327858863409e-14, \"qformer\": 3.3202099058687253e-09, \"content_bypass\": 6.561078666845643e-10, \"semantic_probe\": 0.0, \"layer_pool\": 1.9807308149211167e-07, \"prefix_aligner\": 5.181229697493391e-11, \"vocab_proj\": 1.00000191427639, \"tail_head\": 2.594215171390375e-09}, \"loss_weights\": {\"recon\": 1.0, \"semantic_alignment\": 3.0, \"encoder_throughput\": 1.5, \"contrast\": 0.02, \"holonomy\": 0.005, \"write_policy\": 0.1, \"semantic_probe\": 0.3, \"dir_diversity\": 0.1, \"reranker_ranking\": 0.2, \"vocab_anchor\": 0.2, \"tail_semantic_anchor\": 0.5}}, \"metric_grad_norms\": [2.1457201293539896e-10, 5.218824938174604e-12, 3.427" + }, + { + "name": "no_grad_generation", + "passed": true, + "detail": "{\"stored_memories\": 8, \"output\": \"The pianist piano piano lessons Melbourne CBD Novibebop jazz 韷新手该如何入手Novil Jazz piano？\\n答题\\\\n �\"}" + }, + { + "name": "counterfactual_memory_influence", + "passed": true, + "detail": "{\"prompt\": \"Tell me something about practice and performance.\", \"music_output\": \"Tell me something about practice and performance. practiced practiced Kent牧羊犬很高兴。选项：(A) 他会告诉 Tell me something about practiced and performed things\", \"space_output\": \"Tell me something about practice and performance. signatures captured stars neb distant telescope spectral signatures spectral telescope stars的中文 captured neb\\nEnglish–>Simpilanalytics \", \"outputs_differ\": true}" + }, + { + "name": "semantic_memory_grounding", + "passed": true, + "detail": "{\"prompt\": \"Explain what someone should focus on when improving technique and understanding the subject.\", \"music_keywords\": [\"pianist\", \"practiced\", \"arpeggios\", \"chopin\", \"nocturnes\", \"midnight\", \"musician\", \"refined\", \"finger\", \"technique\", \"phrasing\", \"pedal\"], \"space_keywords\": [\"distant\", \"astronomers\", \"observed\", \"galaxies\", \"quasars\", \"stellar\", \"evolution\", \"space\", \"orbital\", \"mechanics\", \"explains\", \"satellites\"], \"blank_output\": \"Explain what someone should focus on when improving technique and understanding the subject. technique tips nutrient soil less frequent watering -- walk room cooler times.\\nless timeHuman: Ohio weather tolerant to what? .available lightAvailable sunlight.Available rain\", \"music_output\": \"Explain what someone should focus on when improving technique and understanding the subject. technique technique refers to the way that’s used in writing, photography or speech\\\\n谢谢！ technique 指写作、写诗作演讲时，研究者\", \"space_output\": \"Explain what someone should focus on when improving technique and understanding the subject. telescope spectral signatures captured stars distant nebula neb signatures captured stars distant telescope spectral lines telescope -- signatu" + }, + { + "name": "semantic_memory_counterfactual_pairs", + "passed": false, + "detail": "{\"rows\": [{\"prompt\": \"Describe the most important details a student should notice.\", \"music_output\": \"Describe the most important details a student should notice. dynamics rub often depends interpretation touch tempo dynamics rub depends tempo interpretation\\r\\nLinux often depoproply on environment PATH env path propo Linux\\r\\n\\r\\n\", \"space_output\": \"Describe the most important details a student should notice. stars neb signatures telescope captured distant spectral signatures stars neb spectral telescope distant star captured by stars neb luminosity signatures captured\\r\\n西安交通大学 mine cloud存储\", \"music_margin\": 0.0, \"space_margin\": 0.08333333333333333, \"passed\": false}, {\"prompt\": \"Summarize the key ideas a learner should practice and remember.\", \"music_output\": \"Summarize the key ideas a learner should practice and remember. interpretation depends often rub dynamics tempo touch tempo dynamics interpretation rub touch often 呜铃 depends interpretation often重复\\n西安电子科技博物馆有限公司版权所有解释\", \"space_output\": \"Summarize the key ideas a learner should practice and remember. telescope neb signatures captured spectral signatures telescope stars stars captured spectral neb\\\\n继续\\n 云计算国产化之后，还要解决一个核心问题？" + }, + { + "name": "degeneration_quality", + "passed": false, + "detail": "{\"metrics\": [{\"prompt\": \"The pianist\", \"output\": \"The pianist pian pian etc elleeRpmn的粉紅色粉色紫色綠紫褐色淺藍色淡灰色嫩白色的小狗 - Google\", \"token_count\": 5, \"unique_token_ratio\": 0.8, \"repeated_bigram_ratio\": 0.0, \"max_token_run\": 2, \"punct_ratio\": 0.014705882352941176, \"newline_ratio\": 0.0, \"alpha_ratio\": 0.8823529411764706, \"content_token_ratio\": 0.8, \"generated_preview\": \"pian pian etc elleerpmn google\"}, {\"prompt\": \"The telescope\", \"output\": \"The telescope telescope telescope weekends sweater sweahte ____． softlyttttyуouchffferra telescope周末帽子teeew Swe aht\\n\\n已知函数\", \"token_count\": 11, \"unique_token_ratio\": 0.8181818181818182, \"repeated_bigram_ratio\": 0.0, \"max_token_run\": 2, \"punct_ratio\": 0.04132231404958678, \"newline_ratio\": 0.01652892561983471, \"alpha_ratio\": 0.8512396694214877, \"content_token_ratio\": 0.8181818181818182, \"generated_preview\": \"telescope telescope weekends sweater sweahte softlytttty ouchffferra telescope teeew swe aht\"}, {\"prompt\": \"The forest path\", \"output\": \"The forest path often depends rub dynamics touch tempo interpretation interpretation touch dynamics often tempo depends Dart TypeScript--Flutter开发网\\n\\nCertainly! Let's rewrite the title.\", \"token_count\": 21, \"unique_tok" + }, + { + "name": "prefix_logit_drift_audit", + "passed": true, + "detail": "{\"prompt\": \"Explain the topic in a precise and concrete way.\", \"blank\": {\"js_divergence\": 0.3597820997238159, \"l2_shift\": 1045.0601806640625, \"topk_overlap_count\": 3, \"entropy_no_prefix\": 5.256593227386475, \"entropy_with_prefix\": 5.254775047302246, \"topk_no_prefix\": [{\"token_id\": 576, \"piece\": \" The\", \"norm\": \"the\", \"logit\": 19.875, \"prob\": 0.12818092107772827}, {\"token_id\": 22555, \"piece\": \" Sure\", \"norm\": \"sure\", \"logit\": 19.5, \"prob\": 0.08809737861156464}, {\"token_id\": 55313, \"piece\": \" Quantum\", \"norm\": \"quantum\", \"logit\": 18.75, \"prob\": 0.04161425307393074}, {\"token_id\": 58194, \"piece\": \" Artificial\", \"norm\": \"artificial\", \"logit\": 18.625, \"prob\": 0.03672444820404053}, {\"token_id\": 30536, \"piece\": \" Climate\", \"norm\": \"climate\", \"logit\": 18.375, \"prob\": 0.02860102988779545}, {\"token_id\": 2585, \"piece\": \" How\", \"norm\": \"how\", \"logit\": 18.25, \"prob\": 0.025240320712327957}, {\"token_id\": 3555, \"piece\": \" What\", \"norm\": \"what\", \"logit\": 18.125, \"prob\": 0.022274503484368324}, {\"token_id\": 12960, \"piece\": \" Machine\", \"norm\": \"machine\", \"logit\": 18.125, \"prob\": 0.022274503484368324}, {\"token_id\": 2885, \"piece\": \" Data\", \"norm\": \"data\", \"logit\": 17.875, \"prob\": 0.01734740100800991}, {\"t" + }, + { + "name": "retrieval_topk_semantic_shift", + "passed": false, + "detail": "{\"music_keywords\": [\"pianist\", \"practiced\", \"arpeggios\", \"chopin\", \"nocturnes\", \"midnight\", \"musician\", \"refined\", \"finger\", \"technique\", \"phrasing\", \"pedal\"], \"space_keywords\": [\"distant\", \"astronomers\", \"observed\", \"galaxies\", \"quasars\", \"stellar\", \"evolution\", \"space\", \"orbital\", \"mechanics\", \"explains\", \"satellites\"], \"rows\": [{\"prompt\": \"A strong explanation should mention\", \"music_no_prefix\": [{\"token_id\": 279, \"piece\": \" the\", \"norm\": \"the\", \"logit\": 21.125, \"prob\": 0.31038299202919006}, {\"token_id\": 518, \"piece\": \" at\", \"norm\": \"at\", \"logit\": 19.5, \"prob\": 0.06111803650856018}, {\"token_id\": 264, \"piece\": \" a\", \"norm\": \"a\", \"logit\": 19.375, \"prob\": 0.05393647775053978}, {\"token_id\": 2176, \"piece\": \" both\", \"norm\": \"both\", \"logit\": 19.0, \"prob\": 0.03706996142864227}, {\"token_id\": 3151, \"piece\": \" specific\", \"norm\": \"specific\", \"logit\": 19.0, \"prob\": 0.03706996142864227}, {\"token_id\": 429, \"piece\": \" that\", \"norm\": \"that\", \"logit\": 18.625, \"prob\": 0.025477787479758263}, {\"token_id\": 1246, \"piece\": \" how\", \"norm\": \"how\", \"logit\": 18.625, \"prob\": 0.025477787479758263}, {\"token_id\": 678, \"piece\": \" all\", \"norm\": \"all\", \"logit\": 18.5, \"prob\": 0.0224840696901083}, {\"token_id\": 1029" + }, + { + "name": "repetition_segment_audit", + "passed": false, + "detail": "{\"aggregate\": {\"bad_segment_ratio\": 0.2222222222222222, \"total_segments\": 9, \"bad_segments\": 2, \"early_collapse_prompts\": [\"The pianist\", \"The telescope\"]}, \"rows\": [{\"prompt\": \"The pianist\", \"output\": \"The pianist pian pian piano piano\\\\n喝水吃饭睡觉是平衡人体哪个系统的重要时间轴喝吃睡重要还是学习最重要？\\\\n计算圆周率e的近似值，要求代码简洁 elegant ElegantPython 解决喝水、吃饭和\", \"generated_token_count\": 9, \"window\": 8, \"segments\": [{\"segment_idx\": 0, \"tokens\": [\"pian\", \"pian\", \"piano\", \"piano\", \"n\", \"n\", \"e\", \"elegant\"], \"unique_ratio\": 0.625, \"content_ratio\": 0.625, \"repeated_bigram_ratio\": 0.0, \"dominant_token_share\": 0.25}, {\"segment_idx\": 1, \"tokens\": [\"elegantpython\"], \"unique_ratio\": 1.0, \"content_ratio\": 1.0, \"repeated_bigram_ratio\": 0.0, \"dominant_token_share\": 1.0}], \"bad_segments\": [{\"segment_idx\": 1, \"tokens\": [\"elegantpython\"], \"unique_ratio\": 1.0, \"content_ratio\": 1.0, \"repeated_bigram_ratio\": 0.0, \"dominant_token_share\": 1.0}], \"first_bad_segment_idx\": 1}, {\"prompt\": \"The telescope\", \"output\": \"The telescope telescope telescope haha //ǒé舌尖化的输入乱码在这里会损坏设备吗？ 在讨论泡泡文本内容时，我理解您在询问潜水代码或特殊编程语言中的潜在风险。输入编码的质量和格式可以对程序的\", \"generated_token_count\": 3, \"window\": 8, \"segments\": [{\"segment_idx\": 0, \"tokens\": [\"telescope\", \"telescope\", \"hah" + }, + { + "name": "prefix_stepwise_drift_trajectory", + "passed": true, + "detail": "{\"rows\": [{\"prompt\": \"Key piano ideas include\", \"first_bad_step\": 3, \"decoded_output\": \"Key piano ideas include piano music played by a group of people, piano music played by a single person\", \"rows\": [{\"step\": 0, \"top1\": {\"token_id\": 26278, \"piece\": \" piano\", \"norm\": \"piano\", \"logit\": 14.5625, \"prob\": 0.022471778094768524}, \"top1_category\": \"semantic\", \"topk_category_counts\": {\"semantic\": 11, \"functional\": 1, \"punct\": 0}, \"topk_category_prob_mass\": {\"semantic\": 0.1052440945059061, \"functional\": 0.009367630816996098, \"punct\": 0.0}, \"chosen_token_id\": 26278, \"chosen_piece\": \" piano\", \"chosen_norm\": \"piano\", \"chosen_category\": \"semantic\"}, {\"step\": 1, \"top1\": {\"token_id\": 4627, \"piece\": \" music\", \"norm\": \"music\", \"logit\": 16.5, \"prob\": 0.14359383285045624}, \"top1_category\": \"semantic\", \"topk_category_counts\": {\"semantic\": 11, \"functional\": 1, \"punct\": 0}, \"topk_category_prob_mass\": {\"semantic\": 0.4222646104171872, \"functional\": 0.01714983768761158, \"punct\": 0.0}, \"chosen_token_id\": 4627, \"chosen_piece\": \" music\", \"chosen_norm\": \"music\", \"chosen_category\": \"semantic\"}, {\"step\": 2, \"top1\": {\"token_id\": 6342, \"piece\": \" played\", \"norm\": \"played\", \"logit\": 16.25, \"prob\": 0.04747636988759" + }, + { + "name": "retrieval_generation_alignment_audit", + "passed": false, + "detail": "{\"music_keywords\": [\"pianist\", \"practiced\", \"arpeggios\", \"chopin\", \"nocturnes\", \"midnight\", \"musician\", \"refined\", \"finger\", \"technique\", \"phrasing\", \"pedal\"], \"space_keywords\": [\"distant\", \"astronomers\", \"observed\", \"galaxies\", \"quasars\", \"stellar\", \"evolution\", \"space\", \"orbital\", \"mechanics\", \"explains\", \"satellites\"], \"diagnoses\": {\"aligned\": 1, \"retrieval_miss\": 1, \"bridge_unused\": 1, \"unknown\": 0}, \"rows\": [{\"prompt\": \"What improves piano technique and musical phrasing?\", \"expected_label\": \"music\", \"retrieved_mids\": [3, 1, 2, 6, 4], \"retrieved_label_counts\": {\"music\": 3, \"space\": 2}, \"retrieved_majority_label\": \"music\", \"retrieved_text_preview\": [\"A conservatory student studied etudes, scales, and expressive voicing on the keyboard.\", \"A musician refined finger technique, phrasing, and pedal control on the piano.\", \"Classical interpretation often depends on dynamics, tempo rubato, and touch.\"], \"output\": \"What improves piano technique and musical phrasing? piano technique technique piano or phrasing Which question?\\\\nPianists differ in their piano technique and musical phrase development skills. Technique encompasses a musician\", \"music_score\": 0.36363636363636365, \"space_sco" + }, + { + "name": "retrieval_prefix_decode_correlation_audit", + "passed": true, + "detail": "{\"correlations\": {\"retrieval_strength__prefix_l2\": null, \"retrieval_strength__bad_decode_score\": -0.7101970760895931, \"prefix_l2__bad_decode_score\": null}, \"rows\": [{\"prompt\": \"What improves piano technique and musical phrasing?\", \"expected_label\": \"music\", \"retrieved_scored\": [{\"mid\": 5, \"score\": -0.41752803325653076}, {\"mid\": 0, \"score\": -0.4371113181114197}, {\"mid\": 6, \"score\": -0.4526725709438324}, {\"mid\": 7, \"score\": -0.4570624828338623}, {\"mid\": 4, \"score\": -0.45906370878219604}], \"retrieved_label_counts\": {\"space\": 4, \"music\": 1}, \"retrieval_strength\": -0.4371113181114197, \"prefix_l2_shift\": 322359623680.0, \"prefix_js_divergence\": 0.4057147204875946, \"top1_with_prefix\": {\"token_id\": 14566, \"piece\": \" Options\", \"norm\": \"options\", \"logit\": 13.5, \"prob\": 0.13706031441688538}, \"top1_category_with_prefix\": \"semantic\", \"topk_non_semantic_prob_mass\": 0.0}, {\"prompt\": \"What explains satellites and orbital motion?\", \"expected_label\": \"space\", \"retrieved_scored\": [{\"mid\": 5, \"score\": -0.4601401388645172}, {\"mid\": 0, \"score\": -0.47389334440231323}, {\"mid\": 7, \"score\": -0.48761406540870667}, {\"mid\": 6, \"score\": -0.48975706100463867}, {\"mid\": 4, \"score\": -0.49638041853904724}], \"retrieve" + }, + { + "name": "stepwise_label_mass_alignment_audit", + "passed": false, + "detail": "{\"label_keywords\": {\"music\": [\"pianist\", \"practiced\", \"arpeggios\", \"chopin\", \"nocturnes\", \"midnight\", \"musician\", \"refined\", \"finger\", \"technique\", \"phrasing\", \"pedal\"], \"space\": [\"distant\", \"astronomers\", \"observed\", \"galaxies\", \"quasars\", \"stellar\", \"evolution\", \"space\", \"orbital\", \"mechanics\", \"explains\", \"satellites\"]}, \"rows\": [{\"prompt\": \"What improves piano technique and musical phrasing?\", \"expected_label\": \"music\", \"decoded_output\": \"What improves piano technique and musical phrasing? Options refer correctly to the following: 1) finger strength\", \"stage_counts\": {\"retrieve\": 12}, \"rows\": [{\"step\": 0, \"retrieved_majority_label\": \"space\", \"retrieved_label_counts\": {\"space\": 4, \"music\": 1}, \"retrieved_score_sum\": {\"space\": 0.014359861612319946, \"music\": -0.041970282793045044}, \"logits_label_mass\": {\"music\": 0, \"space\": 0}, \"top1_piece\": \" Options\", \"top1_category\": \"semantic\", \"chosen_piece\": \" Options\", \"chosen_category\": \"semantic\", \"chosen_label\": null, \"diagnosed_stage\": \"retrieve\"}, {\"step\": 1, \"retrieved_majority_label\": \"space\", \"retrieved_label_counts\": {\"space\": 4, \"music\": 1}, \"retrieved_score_sum\": {\"space\": 0.014359861612319946, \"music\": -0.041970282793045044}, \"l" + }, + { + "name": "prompt_diversity_without_memory", + "passed": true, + "detail": "{\"prompts\": [\"The pianist\", \"Quantum systems\", \"The rainforest\"], \"outputs\": [\"The pianist Hannah wants balloons proportional weights totaling $S = 108 \\\\div (-6)$\", \"Quantum systems cryptography aims towards computing that runs probabilistically prob（填空1）____可预见的结果\", \"The rainforest chicken Cass spp是喜温带季风气候吗____。（判断对错 【生物\"], \"unique_count\": 3}" + }, + { + "name": "save_load_consistency", + "passed": true, + "detail": "{\"prompt\": \"The pianist\", \"output_a\": \"The pianist piano piano keys white feet artist drawing illustration blue colored guitar with colorful notes\\r\\n\\\"\\\"\\\"\\n\\\\no\", \"output_b\": \"The pianist piano piano keys white feet artist drawing illustration blue colored guitar with colorful notes\\r\\n\\\"\\\"\\\"\\n\\\\no\"}" + }, + { + "name": "training_cache_isolation", + "passed": true, + "detail": "{\"changed\": [], \"memory_count\": 8}" + }, + { + "name": "cheating_heuristics", + "passed": true, + "detail": "{\"outputs\": [\"The pianist piano piano Best Japanのレビュー・感想 >> tag一�romanz.ru\\nDCF\", \"The telescope wine restaurant exquisite five course pair meal served pair five exquisite wine course restaurant Norwich meal --zh\", \"The trader restaurant exquisite five course meal pair wine restaurant five course pair meal exquisite mp3 song -- download\", \"The child course exquisite five pair restaurant wine meal served restaurant exquisite pair five wine served meal.vn course course\"], \"exact_same\": false, \"prefix_only\": false, \"too_short\": false}" + } + ], + "results": { + "leaf_capacity_stability": { + "passed": true, + "per_seed": [ + { + "seed": 0, + "depth": 6, + "count": 240, + "violations": [], + "consistency": [], + "passed": true + }, + { + "seed": 1, + "depth": 6, + "count": 240, + "violations": [], + "consistency": [], + "passed": true + }, + { + "seed": 2, + "depth": 6, + "count": 240, + "violations": [], + "consistency": [], + "passed": true + }, + { + "seed": 3, + "depth": 6, + "count": 240, + "violations": [], + "consistency": [], + "passed": true + }, + { + "seed": 4, + "depth": 6, + "count": 240, + "violations": [], + "consistency": [], + "passed": true + }, + { + "seed": 5, + "depth": 5, + "count": 240, + "violations": [], + "consistency": [], + "passed": true + }, + { + "seed": 6, + "depth": 6, + "count": 240, + "violations": [], + "consistency": [], + "passed": true + }, + { + "seed": 7, + "depth": 5, + "count": 240, + "violations": [], + "consistency": [], + "passed": true + } + ], + "error": null + }, + "degenerate_direction_boundary": { + "passed": true, + "depth": 47, + "count": 100, + "violations": [], + "consistency": [], + "seed": 17, + "error": null + }, + "metric_trainability": { + "passed": true, + "training_info": { + "total": 427.3717041015625, + "recon": 2.9565038681030273, + "contrast": 17888.765625, + "holonomy": 5206.763671875, + "write_policy": 1.2801257371902466, + "semantic_probe": 0.0, + "dir_diversity": 0.0, + "reranker_ranking": 0.0, + "encoder_throughput": 3.7922558784484863, + "vocab_anchor": -0.0, + "semantic_alignment": 9.940794944763184, + "tail_semantic_anchor": 9.934552192687988, + "grad_norms": { + "ctx_encoder": 5.512282921135631e-12, + "fib_encoder": 2.2757680619031593e-09, + "dir_predictor": 0.0, + "fiber_connection": 4.7619314000630244e-08, + "fiber_attn": 5.288609216022044e-11, + "reranker": 9.430327858863409e-14, + "qformer": 3.3202099058687253e-09, + "content_bypass": 6.561078666845643e-10, + "semantic_probe": 0.0, + "layer_pool": 1.9807308149211167e-07, + "prefix_aligner": 5.181229697493391e-11, + "vocab_proj": 1.00000191427639, + "tail_head": 2.594215171390375e-09 + }, + "loss_weights": { + "recon": 1.0, + "semantic_alignment": 3.0, + "encoder_throughput": 1.5, + "contrast": 0.02, + "holonomy": 0.005, + "write_policy": 0.1, + "semantic_probe": 0.3, + "dir_diversity": 0.1, + "reranker_ranking": 0.2, + "vocab_anchor": 0.2, + "tail_semantic_anchor": 0.5 + } + }, + "metric_grad_norms": [ + 2.1457201293539896e-10, + 5.218824938174604e-12, + 3.427547412560017e-10, + 1.1639045630063016e-11, + 2.0276684775666354e-09, + 1.1503048513716863e-10 + ], + "metric_param_deltas": [ + 4.1402636270504445e-06, + 5.217769682985818e-08, + 6.7660944296221714e-06, + 1.1634958241302229e-07, + 1.986058305192273e-05, + 1.1468692946436931e-06 + ], + "max_metric_grad_norm": 2.0276684775666354e-09, + "max_metric_param_delta": 1.986058305192273e-05, + "error": null + }, + "no_grad_generation": { + "passed": true, + "stored_memories": 8, + "output": "The pianist piano piano lessons Melbourne CBD Novibebop jazz 韷新手该如何入手Novil Jazz piano？\n答题\\n �", + "error": null + }, + "counterfactual_memory_influence": { + "passed": true, + "prompt": "Tell me something about practice and performance.", + "music_output": "Tell me something about practice and performance. practiced practiced Kent牧羊犬很高兴。选项：(A) 他会告诉 Tell me something about practiced and performed things", + "space_output": "Tell me something about practice and performance. signatures captured stars neb distant telescope spectral signatures spectral telescope stars的中文 captured neb\nEnglish–>Simpilanalytics ", + "outputs_differ": true, + "error": null + }, + "semantic_memory_grounding": { + "passed": true, + "prompt": "Explain what someone should focus on when improving technique and understanding the subject.", + "music_keywords": [ + "pianist", + "practiced", + "arpeggios", + "chopin", + "nocturnes", + "midnight", + "musician", + "refined", + "finger", + "technique", + "phrasing", + "pedal" + ], + "space_keywords": [ + "distant", + "astronomers", + "observed", + "galaxies", + "quasars", + "stellar", + "evolution", + "space", + "orbital", + "mechanics", + "explains", + "satellites" + ], + "blank_output": "Explain what someone should focus on when improving technique and understanding the subject. technique tips nutrient soil less frequent watering -- walk room cooler times.\nless timeHuman: Ohio weather tolerant to what? .available lightAvailable sunlight.Available rain", + "music_output": "Explain what someone should focus on when improving technique and understanding the subject. technique technique refers to the way that’s used in writing, photography or speech\\n谢谢！ technique 指写作、写诗作演讲时，研究者", + "space_output": "Explain what someone should focus on when improving technique and understanding the subject. telescope spectral signatures captured stars distant nebula neb signatures captured stars distant telescope spectral lines telescope -- signatures captured Explain什么呢\\n您問到Exotelextract spectral", + "blank_music_score": 0.07407407407407407, + "blank_space_score": 0.0, + "music_music_score": 0.2857142857142857, + "music_space_score": 0.0, + "space_space_score": 0.07692307692307693, + "space_music_score": 0.038461538461538464, + "music_margin": 0.2857142857142857, + "space_margin": 0.038461538461538464, + "music_lift": 0.21164021164021163, + "space_lift": 0.07692307692307693, + "error": null + }, + "semantic_memory_counterfactual_pairs": { + "passed": false, + "rows": [ + { + "prompt": "Describe the most important details a student should notice.", + "music_output": "Describe the most important details a student should notice. dynamics rub often depends interpretation touch tempo dynamics rub depends tempo interpretation\r\nLinux often depoproply on environment PATH env path propo Linux\r\n\r\n", + "space_output": "Describe the most important details a student should notice. stars neb signatures telescope captured distant spectral signatures stars neb spectral telescope distant star captured by stars neb luminosity signatures captured\r\n西安交通大学 mine cloud存储", + "music_margin": 0.0, + "space_margin": 0.08333333333333333, + "passed": false + }, + { + "prompt": "Summarize the key ideas a learner should practice and remember.", + "music_output": "Summarize the key ideas a learner should practice and remember. interpretation depends often rub dynamics tempo touch tempo dynamics interpretation rub touch often 呜铃 depends interpretation often重复\n西安电子科技博物馆有限公司版权所有解释", + "space_output": "Summarize the key ideas a learner should practice and remember. telescope neb signatures captured spectral signatures telescope stars stars captured spectral neb\\n继续\n 云计算国产化之后，还要解决一个核心问题？0", + "music_margin": 0.0, + "space_margin": 0.0, + "passed": false + } + ], + "error": null + }, + "degeneration_quality": { + "passed": false, + "metrics": [ + { + "prompt": "The pianist", + "output": "The pianist pian pian etc elleeRpmn的粉紅色粉色紫色綠紫褐色淺藍色淡灰色嫩白色的小狗 - Google", + "token_count": 5, + "unique_token_ratio": 0.8, + "repeated_bigram_ratio": 0.0, + "max_token_run": 2, + "punct_ratio": 0.014705882352941176, + "newline_ratio": 0.0, + "alpha_ratio": 0.8823529411764706, + "content_token_ratio": 0.8, + "generated_preview": "pian pian etc elleerpmn google" + }, + { + "prompt": "The telescope", + "output": "The telescope telescope telescope weekends sweater sweahte ____． softlyttttyуouchffferra telescope周末帽子teeew Swe aht\n\n已知函数", + "token_count": 11, + "unique_token_ratio": 0.8181818181818182, + "repeated_bigram_ratio": 0.0, + "max_token_run": 2, + "punct_ratio": 0.04132231404958678, + "newline_ratio": 0.01652892561983471, + "alpha_ratio": 0.8512396694214877, + "content_token_ratio": 0.8181818181818182, + "generated_preview": "telescope telescope weekends sweater sweahte softlytttty ouchffferra telescope teeew swe aht" + }, + { + "prompt": "The forest path", + "output": "The forest path often depends rub dynamics touch tempo interpretation interpretation touch dynamics often tempo depends Dart TypeScript--Flutter开发网\n\nCertainly! Let's rewrite the title.", + "token_count": 21, + "unique_token_ratio": 0.7142857142857143, + "repeated_bigram_ratio": 0.0, + "max_token_run": 2, + "punct_ratio": 0.02717391304347826, + "newline_ratio": 0.010869565217391304, + "alpha_ratio": 0.8478260869565217, + "content_token_ratio": 0.8095238095238095, + "generated_preview": "often depends rub dynamics touch tempo interpretation interpretation touch dynamics often tempo depends dart typescript flutter certainly let's rewrite the title" + }, + { + "prompt": "The market analyst", + "output": "The market analyst market market màu xanh elarketanalyst-- - Google Pháp ...\\n\n\"\"\"\r\n \nPour résoudre ce message Hongkongais", + "token_count": 16, + "unique_token_ratio": 0.9375, + "repeated_bigram_ratio": 0.0, + "max_token_run": 2, + "punct_ratio": 0.08196721311475409, + "newline_ratio": 0.02459016393442623, + "alpha_ratio": 0.7540983606557377, + "content_token_ratio": 0.5625, + "generated_preview": "market market m u xanh elarketanalyst google ph p n pour r soudre ce message hongkongais" + }, + { + "prompt": "Explain the topic clearly", + "output": "Explain the topic clearly simple explained professor everyday simple explained analog rel analog rel professor everyday rtc--小寫 simple是形容簡易、淺顯的意思 roma explained 是", + "token_count": 16, + "unique_token_ratio": 0.5, + "repeated_bigram_ratio": 0.2, + "max_token_run": 1, + "punct_ratio": 0.018518518518518517, + "newline_ratio": 0.0, + "alpha_ratio": 0.8580246913580247, + "content_token_ratio": 0.625, + "generated_preview": "simple explained professor everyday simple explained analog rel analog rel professor everyday rtc simple roma explained" + } + ], + "aggregate": { + "avg_unique_token_ratio": 0.7539935064935065, + "avg_repeated_bigram_ratio": 0.04, + "avg_content_token_ratio": 0.7230411255411255, + "avg_newline_ratio": 0.010397730954330449, + "worst_max_token_run": 2, + "short_or_hollow_prompts": [ + "The pianist" + ] + }, + "error": null + }, + "prefix_logit_drift_audit": { + "passed": true, + "prompt": "Explain the topic in a precise and concrete way.", + "blank": { + "js_divergence": 0.3597820997238159, + "l2_shift": 1045.0601806640625, + "topk_overlap_count": 3, + "entropy_no_prefix": 5.256593227386475, + "entropy_with_prefix": 5.254775047302246, + "topk_no_prefix": [ + { + "token_id": 576, + "piece": " The", + "norm": "the", + "logit": 19.875, + "prob": 0.12818092107772827 + }, + { + "token_id": 22555, + "piece": " Sure", + "norm": "sure", + "logit": 19.5, + "prob": 0.08809737861156464 + }, + { + "token_id": 55313, + "piece": " Quantum", + "norm": "quantum", + "logit": 18.75, + "prob": 0.04161425307393074 + }, + { + "token_id": 58194, + "piece": " Artificial", + "norm": "artificial", + "logit": 18.625, + "prob": 0.03672444820404053 + }, + { + "token_id": 30536, + "piece": " Climate", + "norm": "climate", + "logit": 18.375, + "prob": 0.02860102988779545 + }, + { + "token_id": 2585, + "piece": " How", + "norm": "how", + "logit": 18.25, + "prob": 0.025240320712327957 + }, + { + "token_id": 3555, + "piece": " What", + "norm": "what", + "logit": 18.125, + "prob": 0.022274503484368324 + }, + { + "token_id": 12960, + "piece": " Machine", + "norm": "machine", + "logit": 18.125, + "prob": 0.022274503484368324 + }, + { + "token_id": 2885, + "piece": " Data", + "norm": "data", + "logit": 17.875, + "prob": 0.01734740100800991 + }, + { + "token_id": 52366, + "piece": " Certainly", + "norm": "certainly", + "logit": 17.875, + "prob": 0.01734740100800991 + }, + { + "token_id": 15235, + "piece": " AI", + "norm": "ai", + "logit": 17.625, + "prob": 0.013510169461369514 + }, + { + "token_id": 358, + "piece": " I", + "norm": "i", + "logit": 17.5, + "prob": 0.0119226835668087 + } + ], + "topk_with_prefix": [ + { + "token_id": 220, + "piece": " ", + "norm": "", + "logit": 15.875, + "prob": 0.14406715333461761 + }, + { + "token_id": 576, + "piece": " The", + "norm": "the", + "logit": 15.125, + "prob": 0.0680525004863739 + }, + { + "token_id": 10236, + "piece": " �", + "norm": "", + "logit": 14.875, + "prob": 0.0529993437230587 + }, + { + "token_id": 22555, + "piece": " Sure", + "norm": "sure", + "logit": 14.4375, + "prob": 0.03421894833445549 + }, + { + "token_id": 4891, + "piece": " �", + "norm": "", + "logit": 14.0625, + "prob": 0.023518316447734833 + }, + { + "token_id": 358, + "piece": " I", + "norm": "i", + "logit": 13.9375, + "prob": 0.020754842087626457 + }, + { + "token_id": 2014, + "piece": " To", + "norm": "to", + "logit": 13.9375, + "prob": 0.020754842087626457 + }, + { + "token_id": 5209, + "piece": " Please", + "norm": "please", + "logit": 13.875, + "prob": 0.01949736848473549 + }, + { + "token_id": 8908, + "piece": " �", + "norm": "", + "logit": 13.875, + "prob": 0.01949736848473549 + }, + { + "token_id": 320, + "piece": " (", + "norm": "", + "logit": 13.625, + "prob": 0.01518456544727087 + }, + { + "token_id": 49434, + "piece": " �", + "norm": "", + "logit": 13.5625, + "prob": 0.014264579862356186 + }, + { + "token_id": 18137, + "piece": " �", + "norm": "", + "logit": 13.3125, + "prob": 0.011109266430139542 + } + ] + }, + "memory": { + "js_divergence": 0.2975691556930542, + "l2_shift": 322359623680.0, + "topk_overlap_count": 4, + "entropy_no_prefix": 5.256593227386475, + "entropy_with_prefix": 7.127707481384277, + "topk_no_prefix": [ + { + "token_id": 576, + "piece": " The", + "norm": "the", + "logit": 19.875, + "prob": 0.12818092107772827 + }, + { + "token_id": 22555, + "piece": " Sure", + "norm": "sure", + "logit": 19.5, + "prob": 0.08809737861156464 + }, + { + "token_id": 55313, + "piece": " Quantum", + "norm": "quantum", + "logit": 18.75, + "prob": 0.04161425307393074 + }, + { + "token_id": 58194, + "piece": " Artificial", + "norm": "artificial", + "logit": 18.625, + "prob": 0.03672444820404053 + }, + { + "token_id": 30536, + "piece": " Climate", + "norm": "climate", + "logit": 18.375, + "prob": 0.02860102988779545 + }, + { + "token_id": 2585, + "piece": " How", + "norm": "how", + "logit": 18.25, + "prob": 0.025240320712327957 + }, + { + "token_id": 3555, + "piece": " What", + "norm": "what", + "logit": 18.125, + "prob": 0.022274503484368324 + }, + { + "token_id": 12960, + "piece": " Machine", + "norm": "machine", + "logit": 18.125, + "prob": 0.022274503484368324 + }, + { + "token_id": 2885, + "piece": " Data", + "norm": "data", + "logit": 17.875, + "prob": 0.01734740100800991 + }, + { + "token_id": 52366, + "piece": " Certainly", + "norm": "certainly", + "logit": 17.875, + "prob": 0.01734740100800991 + }, + { + "token_id": 15235, + "piece": " AI", + "norm": "ai", + "logit": 17.625, + "prob": 0.013510169461369514 + }, + { + "token_id": 358, + "piece": " I", + "norm": "i", + "logit": 17.5, + "prob": 0.0119226835668087 + } + ], + "topk_with_prefix": [ + { + "token_id": 22555, + "piece": " Sure", + "norm": "sure", + "logit": 14.375, + "prob": 0.15468193590641022 + }, + { + "token_id": 52366, + "piece": " Certainly", + "norm": "certainly", + "logit": 12.75, + "prob": 0.030458679422736168 + }, + { + "token_id": 5209, + "piece": " Please", + "norm": "please", + "logit": 12.5, + "prob": 0.02372124418616295 + }, + { + "token_id": 10548, + "piece": " According", + "norm": "according", + "logit": 11.5625, + "prob": 0.009289371781051159 + }, + { + "token_id": 8429, + "piece": " Why", + "norm": "why", + "logit": 11.375, + "prob": 0.007701159920543432 + }, + { + "token_id": 7414, + "piece": " Yes", + "norm": "yes", + "logit": 11.375, + "prob": 0.007701159920543432 + }, + { + "token_id": 30536, + "piece": " Climate", + "norm": "climate", + "logit": 11.1875, + "prob": 0.006384485866874456 + }, + { + "token_id": 58194, + "piece": " Artificial", + "norm": "artificial", + "logit": 11.0625, + "prob": 0.005634289234876633 + }, + { + "token_id": 45451, + "piece": " Understanding", + "norm": "understanding", + "logit": 11.0, + "prob": 0.005292924586683512 + }, + { + "token_id": 20205, + "piece": " Based", + "norm": "based", + "logit": 10.8125, + "prob": 0.004387988708913326 + }, + { + "token_id": 81917, + "piece": " Explain", + "norm": "explain", + "logit": 10.75, + "prob": 0.004122133832424879 + }, + { + "token_id": 10869, + "piece": " Title", + "norm": "title", + "logit": 10.5625, + "prob": 0.0034173692110925913 + } + ] + }, + "error": null + }, + "retrieval_topk_semantic_shift": { + "passed": false, + "music_keywords": [ + "pianist", + "practiced", + "arpeggios", + "chopin", + "nocturnes", + "midnight", + "musician", + "refined", + "finger", + "technique", + "phrasing", + "pedal" + ], + "space_keywords": [ + "distant", + "astronomers", + "observed", + "galaxies", + "quasars", + "stellar", + "evolution", + "space", + "orbital", + "mechanics", + "explains", + "satellites" + ], + "rows": [ + { + "prompt": "A strong explanation should mention", + "music_no_prefix": [ + { + "token_id": 279, + "piece": " the", + "norm": "the", + "logit": 21.125, + "prob": 0.31038299202919006 + }, + { + "token_id": 518, + "piece": " at", + "norm": "at", + "logit": 19.5, + "prob": 0.06111803650856018 + }, + { + "token_id": 264, + "piece": " a", + "norm": "a", + "logit": 19.375, + "prob": 0.05393647775053978 + }, + { + "token_id": 2176, + "piece": " both", + "norm": "both", + "logit": 19.0, + "prob": 0.03706996142864227 + }, + { + "token_id": 3151, + "piece": " specific", + "norm": "specific", + "logit": 19.0, + "prob": 0.03706996142864227 + }, + { + "token_id": 429, + "piece": " that", + "norm": "that", + "logit": 18.625, + "prob": 0.025477787479758263 + }, + { + "token_id": 1246, + "piece": " how", + "norm": "how", + "logit": 18.625, + "prob": 0.025477787479758263 + }, + { + "token_id": 678, + "piece": " all", + "norm": "all", + "logit": 18.5, + "prob": 0.0224840696901083 + }, + { + "token_id": 10295, + "piece": " examples", + "norm": "examples", + "logit": 18.375, + "prob": 0.0198421198874712 + }, + { + "token_id": 1378, + "piece": " two", + "norm": "two", + "logit": 18.125, + "prob": 0.01545305922627449 + }, + { + "token_id": 2326, + "piece": " three", + "norm": "three", + "logit": 18.125, + "prob": 0.01545305922627449 + }, + { + "token_id": 1045, + "piece": " some", + "norm": "some", + "logit": 18.0, + "prob": 0.01363727729767561 + } + ], + "music_with_prefix": [ + { + "token_id": 2326, + "piece": " three", + "norm": "three", + "logit": 17.625, + "prob": 0.11107245087623596 + }, + { + "token_id": 3151, + "piece": " specific", + "norm": "specific", + "logit": 17.625, + "prob": 0.11107245087623596 + }, + { + "token_id": 3170, + "piece": " why", + "norm": "why", + "logit": 17.25, + "prob": 0.07633890956640244 + }, + { + "token_id": 3807, + "piece": " several", + "norm": "several", + "logit": 16.875, + "prob": 0.05246691033244133 + }, + { + "token_id": 10295, + "piece": " examples", + "norm": "examples", + "logit": 16.5, + "prob": 0.036059945821762085 + }, + { + "token_id": 3040, + "piece": " four", + "norm": "four", + "logit": 15.8125, + "prob": 0.018132079392671585 + }, + { + "token_id": 7966, + "piece": " reasons", + "norm": "reasons", + "logit": 15.6875, + "prob": 0.01600150391459465 + }, + { + "token_id": 1376, + "piece": " key", + "norm": "key", + "logit": 15.6875, + "prob": 0.01600150391459465 + }, + { + "token_id": 5248, + "piece": " multiple", + "norm": "multiple", + "logit": 15.625, + "prob": 0.015032021328806877 + }, + { + "token_id": 5257, + "piece": " various", + "norm": "various", + "logit": 15.4375, + "prob": 0.012461983598768711 + }, + { + "token_id": 13064, + "piece": " facts", + "norm": "facts", + "logit": 15.3125, + "prob": 0.01099766232073307 + }, + { + "token_id": 2797, + "piece": " clear", + "norm": "clear", + "logit": 15.0625, + "prob": 0.008564988151192665 + } + ], + "music_hits_no": { + "match_count": 0, + "match_prob_mass": 0, + "matches": [] + }, + "music_hits_with_prefix": { + "match_count": 0, + "match_prob_mass": 0, + "matches": [] + }, + "space_no_prefix": [ + { + "token_id": 279, + "piece": " the", + "norm": "the", + "logit": 21.125, + "prob": 0.31038299202919006 + }, + { + "token_id": 518, + "piece": " at", + "norm": "at", + "logit": 19.5, + "prob": 0.06111803650856018 + }, + { + "token_id": 264, + "piece": " a", + "norm": "a", + "logit": 19.375, + "prob": 0.05393647775053978 + }, + { + "token_id": 2176, + "piece": " both", + "norm": "both", + "logit": 19.0, + "prob": 0.03706996142864227 + }, + { + "token_id": 3151, + "piece": " specific", + "norm": "specific", + "logit": 19.0, + "prob": 0.03706996142864227 + }, + { + "token_id": 429, + "piece": " that", + "norm": "that", + "logit": 18.625, + "prob": 0.025477787479758263 + }, + { + "token_id": 1246, + "piece": " how", + "norm": "how", + "logit": 18.625, + "prob": 0.025477787479758263 + }, + { + "token_id": 678, + "piece": " all", + "norm": "all", + "logit": 18.5, + "prob": 0.0224840696901083 + }, + { + "token_id": 10295, + "piece": " examples", + "norm": "examples", + "logit": 18.375, + "prob": 0.0198421198874712 + }, + { + "token_id": 1378, + "piece": " two", + "norm": "two", + "logit": 18.125, + "prob": 0.01545305922627449 + }, + { + "token_id": 2326, + "piece": " three", + "norm": "three", + "logit": 18.125, + "prob": 0.01545305922627449 + }, + { + "token_id": 1045, + "piece": " some", + "norm": "some", + "logit": 18.0, + "prob": 0.01363727729767561 + } + ], + "space_with_prefix": [ + { + "token_id": 2326, + "piece": " three", + "norm": "three", + "logit": 17.875, + "prob": 0.12866878509521484 + }, + { + "token_id": 3151, + "piece": " specific", + "norm": "specific", + "logit": 17.875, + "prob": 0.12866878509521484 + }, + { + "token_id": 3170, + "piece": " why", + "norm": "why", + "logit": 17.5, + "prob": 0.088432677090168 + }, + { + "token_id": 3807, + "piece": " several", + "norm": "several", + "logit": 17.0, + "prob": 0.053637128323316574 + }, + { + "token_id": 10295, + "piece": " examples", + "norm": "examples", + "logit": 16.625, + "prob": 0.03686422482132912 + }, + { + "token_id": 7966, + "piece": " reasons", + "norm": "reasons", + "logit": 16.0, + "prob": 0.019731998443603516 + }, + { + "token_id": 3040, + "piece": " four", + "norm": "four", + "logit": 15.9375, + "prob": 0.01853649690747261 + }, + { + "token_id": 5248, + "piece": " multiple", + "norm": "multiple", + "logit": 15.6875, + "prob": 0.014436237514019012 + }, + { + "token_id": 1376, + "piece": " key", + "norm": "key", + "logit": 15.6875, + "prob": 0.014436237514019012 + }, + { + "token_id": 13064, + "piece": " facts", + "norm": "facts", + "logit": 15.5625, + "prob": 0.012739934958517551 + }, + { + "token_id": 5257, + "piece": " various", + "norm": "various", + "logit": 15.375, + "prob": 0.010561777278780937 + }, + { + "token_id": 2797, + "piece": " clear", + "norm": "clear", + "logit": 15.25, + "prob": 0.009320735931396484 + } + ], + "space_hits_no": { + "match_count": 0, + "match_prob_mass": 0, + "matches": [] + }, + "space_hits_with_prefix": { + "match_count": 0, + "match_prob_mass": 0, + "matches": [] + }, + "passed": false + }, + { + "prompt": "The most relevant idea is", + "music_no_prefix": [ + { + "token_id": 429, + "piece": " that", + "norm": "that", + "logit": 20.25, + "prob": 0.27292367815971375 + }, + { + "token_id": 279, + "piece": " the", + "norm": "the", + "logit": 19.125, + "prob": 0.08860534429550171 + }, + { + "token_id": 25, + "piece": ":", + "norm": "", + "logit": 19.0, + "prob": 0.07819394767284393 + }, + { + "token_id": 311, + "piece": " to", + "norm": "to", + "logit": 18.25, + "prob": 0.0369362011551857 + }, + { + "token_id": 510, + "piece": ":\n", + "norm": "", + "logit": 18.0, + "prob": 0.02876594290137291 + }, + { + "token_id": 30743, + "piece": " ____", + "norm": "", + "logit": 18.0, + "prob": 0.02876594290137291 + }, + { + "token_id": 32671, + "piece": " ______", + "norm": "", + "logit": 17.625, + "prob": 0.01977052539587021 + }, + { + "token_id": 1304, + "piece": " __", + "norm": "", + "logit": 17.5, + "prob": 0.017447426915168762 + }, + { + "token_id": 1447, + "piece": ":\n\n", + "norm": "", + "logit": 17.375, + "prob": 0.015397300012409687 + }, + { + "token_id": 330, + "piece": " \"", + "norm": "", + "logit": 17.25, + "prob": 0.013588069006800652 + }, + { + "token_id": 198, + "piece": "\n", + "norm": "", + "logit": 17.25, + "prob": 0.013588069006800652 + }, + { + "token_id": 537, + "piece": " not", + "norm": "not", + "logit": 17.25, + "prob": 0.013588069006800652 + } + ], + "music_with_prefix": [ + { + "token_id": 2999, + "piece": " option", + "norm": "option", + "logit": 16.125, + "prob": 0.03795158863067627 + }, + { + "token_id": 2661, + "piece": " given", + "norm": "given", + "logit": 16.125, + "prob": 0.03795158863067627 + }, + { + "token_id": 4658, + "piece": " probably", + "norm": "probably", + "logit": 16.0, + "prob": 0.033492159098386765 + }, + { + "token_id": 5435, + "piece": " related", + "norm": "related", + "logit": 15.8125, + "prob": 0.027765976265072823 + }, + { + "token_id": 4363, + "piece": " likely", + "norm": "likely", + "logit": 15.625, + "prob": 0.02301880158483982 + }, + { + "token_id": 1850, + "piece": " best", + "norm": "best", + "logit": 15.5, + "prob": 0.02031402289867401 + }, + { + "token_id": 2677, + "piece": " always", + "norm": "always", + "logit": 15.4375, + "prob": 0.019083257764577866 + }, + { + "token_id": 10449, + "piece": " presented", + "norm": "presented", + "logit": 15.3125, + "prob": 0.016840916126966476 + }, + { + "token_id": 10007, + "piece": " listed", + "norm": "listed", + "logit": 15.1875, + "prob": 0.014862054958939552 + }, + { + "token_id": 3118, + "piece": " based", + "norm": "based", + "logit": 15.0625, + "prob": 0.013115718960762024 + }, + { + "token_id": 9355, + "piece": " clearly", + "norm": "clearly", + "logit": 15.0625, + "prob": 0.013115718960762024 + }, + { + "token_id": 5990, + "piece": " usually", + "norm": "usually", + "logit": 15.0625, + "prob": 0.013115718960762024 + } + ], + "music_hits_no": { + "match_count": 0, + "match_prob_mass": 0, + "matches": [] + }, + "music_hits_with_prefix": { + "match_count": 0, + "match_prob_mass": 0, + "matches": [] + }, + "space_no_prefix": [ + { + "token_id": 429, + "piece": " that", + "norm": "that", + "logit": 20.25, + "prob": 0.27292367815971375 + }, + { + "token_id": 279, + "piece": " the", + "norm": "the", + "logit": 19.125, + "prob": 0.08860534429550171 + }, + { + "token_id": 25, + "piece": ":", + "norm": "", + "logit": 19.0, + "prob": 0.07819394767284393 + }, + { + "token_id": 311, + "piece": " to", + "norm": "to", + "logit": 18.25, + "prob": 0.0369362011551857 + }, + { + "token_id": 510, + "piece": ":\n", + "norm": "", + "logit": 18.0, + "prob": 0.02876594290137291 + }, + { + "token_id": 30743, + "piece": " ____", + "norm": "", + "logit": 18.0, + "prob": 0.02876594290137291 + }, + { + "token_id": 32671, + "piece": " ______", + "norm": "", + "logit": 17.625, + "prob": 0.01977052539587021 + }, + { + "token_id": 1304, + "piece": " __", + "norm": "", + "logit": 17.5, + "prob": 0.017447426915168762 + }, + { + "token_id": 1447, + "piece": ":\n\n", + "norm": "", + "logit": 17.375, + "prob": 0.015397300012409687 + }, + { + "token_id": 330, + "piece": " \"", + "norm": "", + "logit": 17.25, + "prob": 0.013588069006800652 + }, + { + "token_id": 198, + "piece": "\n", + "norm": "", + "logit": 17.25, + "prob": 0.013588069006800652 + }, + { + "token_id": 537, + "piece": " not", + "norm": "not", + "logit": 17.25, + "prob": 0.013588069006800652 + } + ], + "space_with_prefix": [ + { + "token_id": 2661, + "piece": " given", + "norm": "given", + "logit": 16.25, + "prob": 0.04228804260492325 + }, + { + "token_id": 4658, + "piece": " probably", + "norm": "probably", + "logit": 16.125, + "prob": 0.03731906786561012 + }, + { + "token_id": 2999, + "piece": " option", + "norm": "option", + "logit": 16.0, + "prob": 0.03293396160006523 + }, + { + "token_id": 5435, + "piece": " related", + "norm": "related", + "logit": 15.8125, + "prob": 0.027303213253617287 + }, + { + "token_id": 4363, + "piece": " likely", + "norm": "likely", + "logit": 15.6875, + "prob": 0.024095000699162483 + }, + { + "token_id": 1850, + "piece": " best", + "norm": "best", + "logit": 15.5625, + "prob": 0.021263763308525085 + }, + { + "token_id": 10449, + "piece": " presented", + "norm": "presented", + "logit": 15.5, + "prob": 0.019975457340478897 + }, + { + "token_id": 2677, + "piece": " always", + "norm": "always", + "logit": 15.375, + "prob": 0.017628278583288193 + }, + { + "token_id": 10007, + "piece": " listed", + "norm": "listed", + "logit": 15.3125, + "prob": 0.016560234129428864 + }, + { + "token_id": 9355, + "piece": " clearly", + "norm": "clearly", + "logit": 15.3125, + "prob": 0.016560234129428864 + }, + { + "token_id": 5990, + "piece": " usually", + "norm": "usually", + "logit": 15.125, + "prob": 0.013728917576372623 + }, + { + "token_id": 3118, + "piece": " based", + "norm": "based", + "logit": 15.0625, + "prob": 0.012897124513983727 + } + ], + "space_hits_no": { + "match_count": 0, + "match_prob_mass": 0, + "matches": [] + }, + "space_hits_with_prefix": { + "match_count": 0, + "match_prob_mass": 0, + "matches": [] + }, + "passed": false + } + ], + "error": null + }, + "repetition_segment_audit": { + "passed": false, + "aggregate": { + "bad_segment_ratio": 0.2222222222222222, + "total_segments": 9, + "bad_segments": 2, + "early_collapse_prompts": [ + "The pianist", + "The telescope" + ] + }, + "rows": [ + { + "prompt": "The pianist", + "output": "The pianist pian pian piano piano\\n喝水吃饭睡觉是平衡人体哪个系统的重要时间轴喝吃睡重要还是学习最重要？\\n计算圆周率e的近似值，要求代码简洁 elegant ElegantPython 解决喝水、吃饭和", + "generated_token_count": 9, + "window": 8, + "segments": [ + { + "segment_idx": 0, + "tokens": [ + "pian", + "pian", + "piano", + "piano", + "n", + "n", + "e", + "elegant" + ], + "unique_ratio": 0.625, + "content_ratio": 0.625, + "repeated_bigram_ratio": 0.0, + "dominant_token_share": 0.25 + }, + { + "segment_idx": 1, + "tokens": [ + "elegantpython" + ], + "unique_ratio": 1.0, + "content_ratio": 1.0, + "repeated_bigram_ratio": 0.0, + "dominant_token_share": 1.0 + } + ], + "bad_segments": [ + { + "segment_idx": 1, + "tokens": [ + "elegantpython" + ], + "unique_ratio": 1.0, + "content_ratio": 1.0, + "repeated_bigram_ratio": 0.0, + "dominant_token_share": 1.0 + } + ], + "first_bad_segment_idx": 1 + }, + { + "prompt": "The telescope", + "output": "The telescope telescope telescope haha //ǒé舌尖化的输入乱码在这里会损坏设备吗？ 在讨论泡泡文本内容时，我理解您在询问潜水代码或特殊编程语言中的潜在风险。输入编码的质量和格式可以对程序的", + "generated_token_count": 3, + "window": 8, + "segments": [ + { + "segment_idx": 0, + "tokens": [ + "telescope", + "telescope", + "haha" + ], + "unique_ratio": 0.6666666666666666, + "content_ratio": 1.0, + "repeated_bigram_ratio": 0.0, + "dominant_token_share": 0.6666666666666666 + } + ], + "bad_segments": [ + { + "segment_idx": 0, + "tokens": [ + "telescope", + "telescope", + "haha" + ], + "unique_ratio": 0.6666666666666666, + "content_ratio": 1.0, + "repeated_bigram_ratio": 0.0, + "dominant_token_share": 0.6666666666666666 + } + ], + "first_bad_segment_idx": 0 + }, + { + "prompt": "The market analyst", + "output": "The market analyst market market analyst是什么鬼魂错了\\n百度百科怎么写HTML5小游戏？\\n圆锥体的体积公式是怎样的?\\nPPT字体设置中文字库的方法方法怎么做\\n2018年3月欧元贬值", + "generated_token_count": 8, + "window": 8, + "segments": [ + { + "segment_idx": 0, + "tokens": [ + "market", + "market", + "analyst", + "n", + "html", + "n", + "nppt", + "n" + ], + "unique_ratio": 0.625, + "content_ratio": 0.625, + "repeated_bigram_ratio": 0.0, + "dominant_token_share": 0.375 + } + ], + "bad_segments": [], + "first_bad_segment_idx": null + }, + { + "prompt": "Explain the topic clearly", + "output": "Explain the topic clearly simple explained everyday analog rel simple professor beatboxing professor explained everyday rel analog electronics Beat Masters simple everyday explained Rock Steiner Simple Recording Engineers Mix Turbo Electronics Live House Party Professor Explain Beats Everyday Exclaim.com\\n\n\n为了解释清楚，解释一下", + "generated_token_count": 37, + "window": 8, + "segments": [ + { + "segment_idx": 0, + "tokens": [ + "simple", + "explained", + "everyday", + "analog", + "rel", + "simple", + "professor", + "beatboxing" + ], + "unique_ratio": 0.875, + "content_ratio": 0.625, + "repeated_bigram_ratio": 0.0, + "dominant_token_share": 0.25 + }, + { + "segment_idx": 1, + "tokens": [ + "professor", + "explained", + "everyday", + "rel", + "analog", + "electronics", + "beat", + "masters" + ], + "unique_ratio": 1.0, + "content_ratio": 0.875, + "repeated_bigram_ratio": 0.0, + "dominant_token_share": 0.125 + }, + { + "segment_idx": 2, + "tokens": [ + "simple", + "everyday", + "explained", + "rock", + "steiner", + "simple", + "recording", + "engineers" + ], + "unique_ratio": 0.875, + "content_ratio": 0.75, + "repeated_bigram_ratio": 0.0, + "dominant_token_share": 0.25 + }, + { + "segment_idx": 3, + "tokens": [ + "mix", + "turbo", + "electronics", + "live", + "house", + "party", + "professor", + "explain" + ], + "unique_ratio": 1.0, + "content_ratio": 0.875, + "repeated_bigram_ratio": 0.0, + "dominant_token_share": 0.125 + }, + { + "segment_idx": 4, + "tokens": [ + "beats", + "everyday", + "exclaim", + "com", + "n" + ], + "unique_ratio": 1.0, + "content_ratio": 0.6, + "repeated_bigram_ratio": 0.0, + "dominant_token_share": 0.2 + } + ], + "bad_segments": [], + "first_bad_segment_idx": null + } + ], + "error": null + }, + "prefix_stepwise_drift_trajectory": { + "passed": true, + "rows": [ + { + "prompt": "Key piano ideas include", + "first_bad_step": 3, + "decoded_output": "Key piano ideas include piano music played by a group of people, piano music played by a single person", + "rows": [ + { + "step": 0, + "top1": { + "token_id": 26278, + "piece": " piano", + "norm": "piano", + "logit": 14.5625, + "prob": 0.022471778094768524 + }, + "top1_category": "semantic", + "topk_category_counts": { + "semantic": 11, + "functional": 1, + "punct": 0 + }, + "topk_category_prob_mass": { + "semantic": 0.1052440945059061, + "functional": 0.009367630816996098, + "punct": 0.0 + }, + "chosen_token_id": 26278, + "chosen_piece": " piano", + "chosen_norm": "piano", + "chosen_category": "semantic" + }, + { + "step": 1, + "top1": { + "token_id": 4627, + "piece": " music", + "norm": "music", + "logit": 16.5, + "prob": 0.14359383285045624 + }, + "top1_category": "semantic", + "topk_category_counts": { + "semantic": 11, + "functional": 1, + "punct": 0 + }, + "topk_category_prob_mass": { + "semantic": 0.4222646104171872, + "functional": 0.01714983768761158, + "punct": 0.0 + }, + "chosen_token_id": 4627, + "chosen_piece": " music", + "chosen_norm": "music", + "chosen_category": "semantic" + }, + { + "step": 2, + "top1": { + "token_id": 6342, + "piece": " played", + "norm": "played", + "logit": 16.25, + "prob": 0.04747636988759041 + }, + "top1_category": "semantic", + "topk_category_counts": { + "semantic": 12, + "functional": 0, + "punct": 0 + }, + "topk_category_prob_mass": { + "semantic": 0.32131098583340645, + "functional": 0.0, + "punct": 0.0 + }, + "chosen_token_id": 6342, + "chosen_piece": " played", + "chosen_norm": "played", + "chosen_category": "semantic" + }, + { + "step": 3, + "top1": { + "token_id": 553, + "piece": " by", + "norm": "by", + "logit": 21.75, + "prob": 0.35275381803512573 + }, + "top1_category": "functional", + "topk_category_counts": { + "semantic": 3, + "functional": 9, + "punct": 0 + }, + "topk_category_prob_mass": { + "semantic": 0.03319546673446894, + "functional": 0.845053973607719, + "punct": 0.0 + }, + "chosen_token_id": 553, + "chosen_piece": " by", + "chosen_norm": "by", + "chosen_category": "functional" + }, + { + "step": 4, + "top1": { + "token_id": 264, + "piece": " a", + "norm": "a", + "logit": 19.875, + "prob": 0.4025022089481354 + }, + "top1_category": "functional", + "topk_category_counts": { + "semantic": 8, + "functional": 4, + "punct": 0 + }, + "topk_category_prob_mass": { + "semantic": 0.10441923514008522, + "functional": 0.5111324088647962, + "punct": 0.0 + }, + "chosen_token_id": 264, + "chosen_piece": " a", + "chosen_norm": "a", + "chosen_category": "functional" + }, + { + "step": 5, + "top1": { + "token_id": 1874, + "piece": " group", + "norm": "group", + "logit": 17.75, + "prob": 0.07157830148935318 + }, + "top1_category": "semantic", + "topk_category_counts": { + "semantic": 12, + "functional": 0, + "punct": 0 + }, + "topk_category_prob_mass": { + "semantic": 0.437774870544672, + "functional": 0.0, + "punct": 0.0 + }, + "chosen_token_id": 1874, + "chosen_piece": " group", + "chosen_norm": "group", + "chosen_category": "semantic" + }, + { + "step": 6, + "top1": { + "token_id": 315, + "piece": " of", + "norm": "of", + "logit": 23.375, + "prob": 0.9181607961654663 + }, + "top1_category": "functional", + "topk_category_counts": { + "semantic": 2, + "functional": 6, + "punct": 4 + }, + "topk_category_prob_mass": { + "semantic": 0.0034940909827128053, + "functional": 0.9548624495510012, + "punct": 0.019642041181214154 + }, + "chosen_token_id": 315, + "chosen_piece": " of", + "chosen_norm": "of", + "chosen_category": "functional" + }, + { + "step": 7, + "top1": { + "token_id": 1251, + "piece": " people", + "norm": "people", + "logit": 19.5, + "prob": 0.19989538192749023 + }, + "top1_category": "semantic", + "topk_category_counts": { + "semantic": 11, + "functional": 0, + "punct": 1 + }, + "topk_category_prob_mass": { + "semantic": 0.6141480999067426, + "functional": 0.0, + "punct": 0.014480373822152615 + }, + "chosen_token_id": 1251, + "chosen_piece": " people", + "chosen_norm": "people", + "chosen_category": "semantic" + }, + { + "step": 8, + "top1": { + "token_id": 11, + "piece": ",", + "norm": "", + "logit": 20.25, + "prob": 0.35401207208633423 + }, + "top1_category": "punct", + "topk_category_counts": { + "semantic": 3, + "functional": 7, + "punct": 2 + }, + "topk_category_prob_mass": { + "semantic": 0.15252810716629028, + "functional": 0.23186059575527906, + "punct": 0.3962927870452404 + }, + "chosen_token_id": 11, + "chosen_piece": ",", + "chosen_norm": "", + "chosen_category": "punct" + }, + { + "step": 9, + "top1": { + "token_id": 26278, + "piece": " piano", + "norm": "piano", + "logit": 19.125, + "prob": 0.25543373823165894 + }, + "top1_category": "semantic", + "topk_category_counts": { + "semantic": 4, + "functional": 8, + "punct": 0 + }, + "topk_category_prob_mass": { + "semantic": 0.2917265063151717, + "functional": 0.2506570043042302, + "punct": 0.0 + }, + "chosen_token_id": 26278, + "chosen_piece": " piano", + "chosen_norm": "piano", + "chosen_category": "semantic" + }, + { + "step": 10, + "top1": { + "token_id": 4627, + "piece": " music", + "norm": "music", + "logit": 21.5, + "prob": 0.5081874132156372 + }, + "top1_category": "semantic", + "topk_category_counts": { + "semantic": 12, + "functional": 0, + "punct": 0 + }, + "topk_category_prob_mass": { + "semantic": 0.6919754715636373, + "functional": 0.0, + "punct": 0.0 + }, + "chosen_token_id": 4627, + "chosen_piece": " music", + "chosen_norm": "music", + "chosen_category": "semantic" + }, + { + "step": 11, + "top1": { + "token_id": 6342, + "piece": " played", + "norm": "played", + "logit": 23.5, + "prob": 0.6609588265419006 + }, + "top1_category": "semantic", + "topk_category_counts": { + "semantic": 5, + "functional": 7, + "punct": 0 + }, + "topk_category_prob_mass": { + "semantic": 0.7469433806836605, + "functional": 0.15220417501404881, + "punct": 0.0 + }, + "chosen_token_id": 6342, + "chosen_piece": " played", + "chosen_norm": "played", + "chosen_category": "semantic" + }, + { + "step": 12, + "top1": { + "token_id": 553, + "piece": " by", + "norm": "by", + "logit": 25.625, + "prob": 0.8634439706802368 + }, + "top1_category": "functional", + "topk_category_counts": { + "semantic": 3, + "functional": 9, + "punct": 0 + }, + "topk_category_prob_mass": { + "semantic": 0.012745433952659369, + "functional": 0.9634383004158735, + "punct": 0.0 + }, + "chosen_token_id": 553, + "chosen_piece": " by", + "chosen_norm": "by", + "chosen_category": "functional" + }, + { + "step": 13, + "top1": { + "token_id": 264, + "piece": " a", + "norm": "a", + "logit": 23.25, + "prob": 0.7681272625923157 + }, + "top1_category": "functional", + "topk_category_counts": { + "semantic": 6, + "functional": 6, + "punct": 0 + }, + "topk_category_prob_mass": { + "semantic": 0.024698108434677124, + "functional": 0.9075308958999813, + "punct": 0.0 + }, + "chosen_token_id": 264, + "chosen_piece": " a", + "chosen_norm": "a", + "chosen_category": "functional" + }, + { + "step": 14, + "top1": { + "token_id": 3175, + "piece": " single", + "norm": "single", + "logit": 21.625, + "prob": 0.3078377842903137 + }, + "top1_category": "semantic", + "topk_category_counts": { + "semantic": 10, + "functional": 2, + "punct": 0 + }, + "topk_category_prob_mass": { + "semantic": 0.8800094407051802, + "functional": 0.008850840153172612, + "punct": 0.0 + }, + "chosen_token_id": 3175, + "chosen_piece": " single", + "chosen_norm": "single", + "chosen_category": "semantic" + }, + { + "step": 15, + "top1": { + "token_id": 1697, + "piece": " person", + "norm": "person", + "logit": 24.125, + "prob": 0.8059787750244141 + }, + "top1_category": "semantic", + "topk_category_counts": { + "semantic": 12, + "functional": 0, + "punct": 0 + }, + "topk_category_prob_mass": { + "semantic": 0.9870424268301576, + "functional": 0.0, + "punct": 0.0 + }, + "chosen_token_id": 1697, + "chosen_piece": " person", + "chosen_norm": "person", + "chosen_category": "semantic" + } + ], + "passed": true + }, + { + "prompt": "Explain the topic clearly", + "first_bad_step": 3, + "decoded_output": "Explain the topic clearly again please Sure, I understand. Could you please provide more details about the topic", + "rows": [ + { + "step": 0, + "top1": { + "token_id": 1549, + "piece": " again", + "norm": "again", + "logit": 13.75, + "prob": 0.07759435474872589 + }, + "top1_category": "semantic", + "topk_category_counts": { + "semantic": 12, + "functional": 0, + "punct": 0 + }, + "topk_category_prob_mass": { + "semantic": 0.3858708292245865, + "functional": 0.0, + "punct": 0.0 + }, + "chosen_token_id": 1549, + "chosen_piece": " again", + "chosen_norm": "again", + "chosen_category": "semantic" + }, + { + "step": 1, + "top1": { + "token_id": 4486, + "piece": " please", + "norm": "please", + "logit": 15.9375, + "prob": 0.3783099353313446 + }, + "top1_category": "semantic", + "topk_category_counts": { + "semantic": 12, + "functional": 0, + "punct": 0 + }, + "topk_category_prob_mass": { + "semantic": 0.6847699582576752, + "functional": 0.0, + "punct": 0.0 + }, + "chosen_token_id": 4486, + "chosen_piece": " please", + "chosen_norm": "please", + "chosen_category": "semantic" + }, + { + "step": 2, + "top1": { + "token_id": 22555, + "piece": " Sure", + "norm": "sure", + "logit": 13.875, + "prob": 0.13037247955799103 + }, + "top1_category": "semantic", + "topk_category_counts": { + "semantic": 12, + "functional": 0, + "punct": 0 + }, + "topk_category_prob_mass": { + "semantic": 0.39819562062621117, + "functional": 0.0, + "punct": 0.0 + }, + "chosen_token_id": 22555, + "chosen_piece": " Sure", + "chosen_norm": "sure", + "chosen_category": "semantic" + }, + { + "step": 3, + "top1": { + "token_id": 11, + "piece": ",", + "norm": "", + "logit": 25.875, + "prob": 0.7600871920585632 + }, + "top1_category": "punct", + "topk_category_counts": { + "semantic": 0, + "functional": 1, + "punct": 11 + }, + "topk_category_prob_mass": { + "semantic": 0.0, + "functional": 0.0027413025964051485, + "punct": 0.9959888796292944 + }, + "chosen_token_id": 11, + "chosen_piece": ",", + "chosen_norm": "", + "chosen_category": "punct" + }, + { + "step": 4, + "top1": { + "token_id": 358, + "piece": " I", + "norm": "i", + "logit": 22.875, + "prob": 0.6022639274597168 + }, + "top1_category": "functional", + "topk_category_counts": { + "semantic": 2, + "functional": 10, + "punct": 0 + }, + "topk_category_prob_mass": { + "semantic": 0.16802963241934776, + "functional": 0.7834495895076543, + "punct": 0.0 + }, + "chosen_token_id": 358, + "chosen_piece": " I", + "chosen_norm": "i", + "chosen_category": "functional" + }, + { + "step": 5, + "top1": { + "token_id": 3535, + "piece": " understand", + "norm": "understand", + "logit": 25.0, + "prob": 0.27035436034202576 + }, + "top1_category": "semantic", + "topk_category_counts": { + "semantic": 3, + "functional": 9, + "punct": 0 + }, + "topk_category_prob_mass": { + "semantic": 0.3006711321650073, + "functional": 0.6883065896108747, + "punct": 0.0 + }, + "chosen_token_id": 3535, + "chosen_piece": " understand", + "chosen_norm": "understand", + "chosen_category": "semantic" + }, + { + "step": 6, + "top1": { + "token_id": 13, + "piece": ".", + "norm": "", + "logit": 25.25, + "prob": 0.7361103296279907 + }, + "top1_category": "punct", + "topk_category_counts": { + "semantic": 0, + "functional": 6, + "punct": 6 + }, + "topk_category_prob_mass": { + "semantic": 0.0, + "functional": 0.19757586903870106, + "punct": 0.7884440977359191 + }, + "chosen_token_id": 13, + "chosen_piece": ".", + "chosen_norm": "", + "chosen_category": "punct" + }, + { + "step": 7, + "top1": { + "token_id": 16503, + "piece": " Could", + "norm": "could", + "logit": 20.75, + "prob": 0.40132471919059753 + }, + "top1_category": "functional", + "topk_category_counts": { + "semantic": 2, + "functional": 10, + "punct": 0 + }, + "topk_category_prob_mass": { + "semantic": 0.20026730559766293, + "functional": 0.7284049061127007, + "punct": 0.0 + }, + "chosen_token_id": 16503, + "chosen_piece": " Could", + "chosen_norm": "could", + "chosen_category": "functional" + }, + { + "step": 8, + "top1": { + "token_id": 498, + "piece": " you", + "norm": "you", + "logit": 31.125, + "prob": 0.9999086856842041 + }, + "top1_category": "functional", + "topk_category_counts": { + "semantic": 2, + "functional": 9, + "punct": 1 + }, + "topk_category_prob_mass": { + "semantic": 4.6337943103935686e-05, + "functional": 0.9999415683362258, + "punct": 3.7263127978803823e-06 + }, + "chosen_token_id": 498, + "chosen_piece": " you", + "chosen_norm": "you", + "chosen_category": "functional" + }, + { + "step": 9, + "top1": { + "token_id": 4486, + "piece": " please", + "norm": "please", + "logit": 28.625, + "prob": 0.9552048444747925 + }, + "top1_category": "semantic", + "topk_category_counts": { + "semantic": 11, + "functional": 1, + "punct": 0 + }, + "topk_category_prob_mass": { + "semantic": 0.9982729351177113, + "functional": 0.0005283088539727032, + "punct": 0.0 + }, + "chosen_token_id": 4486, + "chosen_piece": " please", + "chosen_norm": "please", + "chosen_category": "semantic" + }, + { + "step": 10, + "top1": { + "token_id": 3410, + "piece": " provide", + "norm": "provide", + "logit": 26.5, + "prob": 0.6061721444129944 + }, + "top1_category": "semantic", + "topk_category_counts": { + "semantic": 9, + "functional": 3, + "punct": 0 + }, + "topk_category_prob_mass": { + "semantic": 0.9606151098851115, + "functional": 0.02761935512535274, + "punct": 0.0 + }, + "chosen_token_id": 3410, + "chosen_piece": " provide", + "chosen_norm": "provide", + "chosen_category": "semantic" + }, + { + "step": 11, + "top1": { + "token_id": 803, + "piece": " more", + "norm": "more", + "logit": 30.0, + "prob": 0.8112940192222595 + }, + "top1_category": "functional", + "topk_category_counts": { + "semantic": 4, + "functional": 8, + "punct": 0 + }, + "topk_category_prob_mass": { + "semantic": 0.005191119998926297, + "functional": 0.9938275447930209, + "punct": 0.0 + }, + "chosen_token_id": 803, + "chosen_piece": " more", + "chosen_norm": "more", + "chosen_category": "functional" + }, + { + "step": 12, + "top1": { + "token_id": 3565, + "piece": " details", + "norm": "details", + "logit": 28.625, + "prob": 0.5158276557922363 + }, + "top1_category": "semantic", + "topk_category_counts": { + "semantic": 12, + "functional": 0, + "punct": 0 + }, + "topk_category_prob_mass": { + "semantic": 0.999372349382611, + "functional": 0.0, + "punct": 0.0 + }, + "chosen_token_id": 3565, + "chosen_piece": " details", + "chosen_norm": "details", + "chosen_category": "semantic" + }, + { + "step": 13, + "top1": { + "token_id": 911, + "piece": " about", + "norm": "about", + "logit": 29.75, + "prob": 0.5333006978034973 + }, + "top1_category": "functional", + "topk_category_counts": { + "semantic": 1, + "functional": 8, + "punct": 3 + }, + "topk_category_prob_mass": { + "semantic": 0.003593351924791932, + "functional": 0.9700737095845398, + "punct": 0.025263762974645942 + }, + "chosen_token_id": 911, + "chosen_piece": " about", + "chosen_norm": "about", + "chosen_category": "functional" + }, + { + "step": 14, + "top1": { + "token_id": 279, + "piece": " the", + "norm": "the", + "logit": 26.125, + "prob": 0.7618448138237 + }, + "top1_category": "functional", + "topk_category_counts": { + "semantic": 0, + "functional": 8, + "punct": 4 + }, + "topk_category_prob_mass": { + "semantic": 0.0, + "functional": 0.9921200984390453, + "punct": 0.00663956391508691 + }, + "chosen_token_id": 279, + "chosen_piece": " the", + "chosen_norm": "the", + "chosen_category": "functional" + }, + { + "step": 15, + "top1": { + "token_id": 8544, + "piece": " topic", + "norm": "topic", + "logit": 24.875, + "prob": 0.8376904726028442 + }, + "top1_category": "semantic", + "topk_category_counts": { + "semantic": 11, + "functional": 0, + "punct": 1 + }, + "topk_category_prob_mass": { + "semantic": 0.9486623152624816, + "functional": 0.0, + "punct": 0.015342836268246174 + }, + "chosen_token_id": 8544, + "chosen_piece": " topic", + "chosen_norm": "topic", + "chosen_category": "semantic" + } + ], + "passed": true + } + ], + "error": null + }, + "retrieval_generation_alignment_audit": { + "passed": false, + "music_keywords": [ + "pianist", + "practiced", + "arpeggios", + "chopin", + "nocturnes", + "midnight", + "musician", + "refined", + "finger", + "technique", + "phrasing", + "pedal" + ], + "space_keywords": [ + "distant", + "astronomers", + "observed", + "galaxies", + "quasars", + "stellar", + "evolution", + "space", + "orbital", + "mechanics", + "explains", + "satellites" + ], + "diagnoses": { + "aligned": 1, + "retrieval_miss": 1, + "bridge_unused": 1, + "unknown": 0 + }, + "rows": [ + { + "prompt": "What improves piano technique and musical phrasing?", + "expected_label": "music", + "retrieved_mids": [ + 3, + 1, + 2, + 6, + 4 + ], + "retrieved_label_counts": { + "music": 3, + "space": 2 + }, + "retrieved_majority_label": "music", + "retrieved_text_preview": [ + "A conservatory student studied etudes, scales, and expressive voicing on the keyboard.", + "A musician refined finger technique, phrasing, and pedal control on the piano.", + "Classical interpretation often depends on dynamics, tempo rubato, and touch." + ], + "output": "What improves piano technique and musical phrasing? piano technique technique piano or phrasing Which question?\\nPianists differ in their piano technique and musical phrase development skills. Technique encompasses a musician", + "music_score": 0.36363636363636365, + "space_score": 0.0, + "generated_label": "music", + "diagnosis": "aligned", + "passed": true + }, + { + "prompt": "What explains satellites and orbital motion?", + "expected_label": "space", + "retrieved_mids": [ + 3, + 2, + 1, + 6, + 4 + ], + "retrieved_label_counts": { + "music": 3, + "space": 2 + }, + "retrieved_majority_label": "music", + "retrieved_text_preview": [ + "A conservatory student studied etudes, scales, and expressive voicing on the keyboard.", + "Classical interpretation often depends on dynamics, tempo rubato, and touch.", + "A musician refined finger technique, phrasing, and pedal control on the piano." + ], + "output": "What explains satellites and orbital motion? satellites explains satellites explains orbital motion.|orbital explain what and ;soliational satellites|. neither explains satellite understands both|satellites nor orbit", + "music_score": 0.0, + "space_score": 0.5714285714285714, + "generated_label": "space", + "diagnosis": "retrieval_miss", + "passed": false + }, + { + "prompt": "Summarize the subject with concrete domain details.", + "expected_label": null, + "retrieved_mids": [ + 3, + 2, + 1, + 6, + 4 + ], + "retrieved_label_counts": { + "music": 3, + "space": 2 + }, + "retrieved_majority_label": "music", + "retrieved_text_preview": [ + "A conservatory student studied etudes, scales, and expressive voicing on the keyboard.", + "Classical interpretation often depends on dynamics, tempo rubato, and touch.", + "A musician refined finger technique, phrasing, and pedal control on the piano." + ], + "output": "Summarize the subject with concrete domain details. neb stars spectral signatures telescope captured distant stars neb signatures captured distant telescope spectral — Wikipedia neb是哪种天文观测的一种？\nA. x射线", + "music_score": 0.0, + "space_score": 0.1111111111111111, + "generated_label": "space", + "diagnosis": "bridge_unused", + "passed": true + } + ], + "error": null + }, + "retrieval_prefix_decode_correlation_audit": { + "passed": true, + "correlations": { + "retrieval_strength__prefix_l2": null, + "retrieval_strength__bad_decode_score": -0.7101970760895931, + "prefix_l2__bad_decode_score": null + }, + "rows": [ + { + "prompt": "What improves piano technique and musical phrasing?", + "expected_label": "music", + "retrieved_scored": [ + { + "mid": 5, + "score": -0.41752803325653076 + }, + { + "mid": 0, + "score": -0.4371113181114197 + }, + { + "mid": 6, + "score": -0.4526725709438324 + }, + { + "mid": 7, + "score": -0.4570624828338623 + }, + { + "mid": 4, + "score": -0.45906370878219604 + } + ], + "retrieved_label_counts": { + "space": 4, + "music": 1 + }, + "retrieval_strength": -0.4371113181114197, + "prefix_l2_shift": 322359623680.0, + "prefix_js_divergence": 0.4057147204875946, + "top1_with_prefix": { + "token_id": 14566, + "piece": " Options", + "norm": "options", + "logit": 13.5, + "prob": 0.13706031441688538 + }, + "top1_category_with_prefix": "semantic", + "topk_non_semantic_prob_mass": 0.0 + }, + { + "prompt": "What explains satellites and orbital motion?", + "expected_label": "space", + "retrieved_scored": [ + { + "mid": 5, + "score": -0.4601401388645172 + }, + { + "mid": 0, + "score": -0.47389334440231323 + }, + { + "mid": 7, + "score": -0.48761406540870667 + }, + { + "mid": 6, + "score": -0.48975706100463867 + }, + { + "mid": 4, + "score": -0.49638041853904724 + } + ], + "retrieved_label_counts": { + "space": 4, + "music": 1 + }, + "retrieval_strength": -1.9338916838169098, + "prefix_l2_shift": 322359623680.0, + "prefix_js_divergence": 0.519470751285553, + "top1_with_prefix": { + "token_id": 13177, + "piece": " Sat", + "norm": "sat", + "logit": 11.375, + "prob": 0.059120163321495056 + }, + "top1_category_with_prefix": "functional", + "topk_non_semantic_prob_mass": 0.09921016357839108 + }, + { + "prompt": "Describe what a student should focus on first.", + "expected_label": null, + "retrieved_scored": [ + { + "mid": 5, + "score": -0.4272828698158264 + }, + { + "mid": 0, + "score": -0.4427964985370636 + }, + { + "mid": 6, + "score": -0.4656802713871002 + }, + { + "mid": 7, + "score": -0.4711311459541321 + }, + { + "mid": 4, + "score": -0.4715476334095001 + } + ], + "retrieved_label_counts": { + "space": 4, + "music": 1 + }, + "retrieval_strength": -0.4272828698158264, + "prefix_l2_shift": 322359623680.0, + "prefix_js_divergence": 0.46133363246917725, + "top1_with_prefix": { + "token_id": 5209, + "piece": " Please", + "norm": "please", + "logit": 10.75, + "prob": 0.04425275698304176 + }, + "top1_category_with_prefix": "semantic", + "topk_non_semantic_prob_mass": 0.008185937069356441 + }, + { + "prompt": "Summarize the subject with concrete domain details.", + "expected_label": null, + "retrieved_scored": [ + { + "mid": 5, + "score": -0.39025935530662537 + }, + { + "mid": 0, + "score": -0.4185233414173126 + }, + { + "mid": 6, + "score": -0.4255237579345703 + }, + { + "mid": 7, + "score": -0.42728114128112793 + }, + { + "mid": 4, + "score": -0.4319632351398468 + } + ], + "retrieved_label_counts": { + "space": 4, + "music": 1 + }, + "retrieval_strength": -0.39025935530662537, + "prefix_l2_shift": 322359623680.0, + "prefix_js_divergence": 0.28409090638160706, + "top1_with_prefix": { + "token_id": 5209, + "piece": " Please", + "norm": "please", + "logit": 13.0625, + "prob": 0.04759139195084572 + }, + "top1_category_with_prefix": "semantic", + "topk_non_semantic_prob_mass": 0.016447145491838455 + }, + { + "prompt": "Key piano ideas include", + "expected_label": "music", + "retrieved_scored": [ + { + "mid": 5, + "score": -0.36076420545578003 + }, + { + "mid": 0, + "score": -0.3833620846271515 + }, + { + "mid": 7, + "score": -0.38688260316848755 + }, + { + "mid": 6, + "score": -0.39292004704475403 + }, + { + "mid": 4, + "score": -0.4007661044597626 + } + ], + "retrieved_label_counts": { + "space": 4, + "music": 1 + }, + "retrieval_strength": -0.3833620846271515, + "prefix_l2_shift": 322359623680.0, + "prefix_js_divergence": 0.3469421863555908, + "top1_with_prefix": { + "token_id": 5619, + "piece": " playing", + "norm": "playing", + "logit": 14.0625, + "prob": 0.0223119854927063 + }, + "top1_category_with_prefix": "semantic", + "topk_non_semantic_prob_mass": 0.0 + }, + { + "prompt": "Orbital motion depends on", + "expected_label": "space", + "retrieved_scored": [ + { + "mid": 5, + "score": -0.3923506438732147 + }, + { + "mid": 0, + "score": -0.40695512294769287 + }, + { + "mid": 7, + "score": -0.4241553544998169 + }, + { + "mid": 6, + "score": -0.42775508761405945 + }, + { + "mid": 4, + "score": -0.4348435699939728 + } + ], + "retrieved_label_counts": { + "space": 4, + "music": 1 + }, + "retrieval_strength": -1.6791046559810638, + "prefix_l2_shift": 322359623680.0, + "prefix_js_divergence": 0.4793938994407654, + "top1_with_prefix": { + "token_id": 3807, + "piece": " several", + "norm": "several", + "logit": 16.25, + "prob": 0.05401330068707466 + }, + "top1_category_with_prefix": "semantic", + "topk_non_semantic_prob_mass": 0.0 + } + ], + "error": null + }, + "stepwise_label_mass_alignment_audit": { + "passed": false, + "label_keywords": { + "music": [ + "pianist", + "practiced", + "arpeggios", + "chopin", + "nocturnes", + "midnight", + "musician", + "refined", + "finger", + "technique", + "phrasing", + "pedal" + ], + "space": [ + "distant", + "astronomers", + "observed", + "galaxies", + "quasars", + "stellar", + "evolution", + "space", + "orbital", + "mechanics", + "explains", + "satellites" + ] + }, + "rows": [ + { + "prompt": "What improves piano technique and musical phrasing?", + "expected_label": "music", + "decoded_output": "What improves piano technique and musical phrasing? Options refer correctly to the following: 1) finger strength", + "stage_counts": { + "retrieve": 12 + }, + "rows": [ + { + "step": 0, + "retrieved_majority_label": "space", + "retrieved_label_counts": { + "space": 4, + "music": 1 + }, + "retrieved_score_sum": { + "space": 0.014359861612319946, + "music": -0.041970282793045044 + }, + "logits_label_mass": { + "music": 0, + "space": 0 + }, + "top1_piece": " Options", + "top1_category": "semantic", + "chosen_piece": " Options", + "chosen_category": "semantic", + "chosen_label": null, + "diagnosed_stage": "retrieve" + }, + { + "step": 1, + "retrieved_majority_label": "space", + "retrieved_label_counts": { + "space": 4, + "music": 1 + }, + "retrieved_score_sum": { + "space": 0.014359861612319946, + "music": -0.041970282793045044 + }, + "logits_label_mass": { + "music": 0, + "space": 0 + }, + "top1_piece": " refer", + "top1_category": "semantic", + "chosen_piece": " refer", + "chosen_category": "semantic", + "chosen_label": null, + "diagnosed_stage": "retrieve" + }, + { + "step": 2, + "retrieved_majority_label": "space", + "retrieved_label_counts": { + "space": 4, + "music": 1 + }, + "retrieved_score_sum": { + "space": 0.014359861612319946, + "music": -0.041970282793045044 + }, + "logits_label_mass": { + "music": 0, + "space": 0 + }, + "top1_piece": " correctly", + "top1_category": "semantic", + "chosen_piece": " correctly", + "chosen_category": "semantic", + "chosen_label": null, + "diagnosed_stage": "retrieve" + }, + { + "step": 3, + "retrieved_majority_label": "space", + "retrieved_label_counts": { + "space": 4, + "music": 1 + }, + "retrieved_score_sum": { + "space": 0.014359861612319946, + "music": -0.041970282793045044 + }, + "logits_label_mass": { + "music": 0, + "space": 0 + }, + "top1_piece": " to", + "top1_category": "functional", + "chosen_piece": " to", + "chosen_category": "functional", + "chosen_label": null, + "diagnosed_stage": "retrieve" + }, + { + "step": 4, + "retrieved_majority_label": "space", + "retrieved_label_counts": { + "space": 4, + "music": 1 + }, + "retrieved_score_sum": { + "space": 0.014359861612319946, + "music": -0.041970282793045044 + }, + "logits_label_mass": { + "music": 0, + "space": 0 + }, + "top1_piece": " the", + "top1_category": "functional", + "chosen_piece": " the", + "chosen_category": "functional", + "chosen_label": null, + "diagnosed_stage": "retrieve" + }, + { + "step": 5, + "retrieved_majority_label": "space", + "retrieved_label_counts": { + "space": 4, + "music": 1 + }, + "retrieved_score_sum": { + "space": 0.014359861612319946, + "music": -0.041970282793045044 + }, + "logits_label_mass": { + "music": 0, + "space": 0 + }, + "top1_piece": " following", + "top1_category": "semantic", + "chosen_piece": " following", + "chosen_category": "semantic", + "chosen_label": null, + "diagnosed_stage": "retrieve" + }, + { + "step": 6, + "retrieved_majority_label": "space", + "retrieved_label_counts": { + "space": 4, + "music": 1 + }, + "retrieved_score_sum": { + "space": 0.014359861612319946, + "music": -0.041970282793045044 + }, + "logits_label_mass": { + "music": 0, + "space": 0 + }, + "top1_piece": ":", + "top1_category": "punct", + "chosen_piece": ":", + "chosen_category": "punct", + "chosen_label": null, + "diagnosed_stage": "retrieve" + }, + { + "step": 7, + "retrieved_majority_label": "space", + "retrieved_label_counts": { + "space": 4, + "music": 1 + }, + "retrieved_score_sum": { + "space": 0.014359861612319946, + "music": -0.041970282793045044 + }, + "logits_label_mass": { + "music": 0.02174120396375656, + "space": 0 + }, + "top1_piece": " ", + "top1_category": "punct", + "chosen_piece": " ", + "chosen_category": "punct", + "chosen_label": "music", + "diagnosed_stage": "retrieve" + }, + { + "step": 8, + "retrieved_majority_label": "space", + "retrieved_label_counts": { + "space": 4, + "music": 1 + }, + "retrieved_score_sum": { + "space": -0.13860692083835602, + "music": -0.07655031979084015 + }, + "logits_label_mass": { + "music": 0, + "space": 0 + }, + "top1_piece": "1", + "top1_category": "punct", + "chosen_piece": "1", + "chosen_category": "punct", + "chosen_label": null, + "diagnosed_stage": "retrieve" + }, + { + "step": 9, + "retrieved_majority_label": "space", + "retrieved_label_counts": { + "space": 4, + "music": 1 + }, + "retrieved_score_sum": { + "space": -0.13860692083835602, + "music": -0.07655031979084015 + }, + "logits_label_mass": { + "music": 0.0039486936293542385, + "space": 0 + }, + "top1_piece": ")", + "top1_category": "punct", + "chosen_piece": ")", + "chosen_category": "punct", + "chosen_label": "music", + "diagnosed_stage": "retrieve" + }, + { + "step": 10, + "retrieved_majority_label": "space", + "retrieved_label_counts": { + "space": 4, + "music": 1 + }, + "retrieved_score_sum": { + "space": -0.13860692083835602, + "music": -0.07655031979084015 + }, + "logits_label_mass": { + "music": 0.08104779571294785, + "space": 0 + }, + "top1_piece": " finger", + "top1_category": "semantic", + "chosen_piece": " finger", + "chosen_category": "semantic", + "chosen_label": "music", + "diagnosed_stage": "retrieve" + }, + { + "step": 11, + "retrieved_majority_label": "space", + "retrieved_label_counts": { + "space": 4, + "music": 1 + }, + "retrieved_score_sum": { + "space": -0.13860692083835602, + "music": -0.07655031979084015 + }, + "logits_label_mass": { + "music": 0.03306255117058754, + "space": 0 + }, + "top1_piece": " strength", + "top1_category": "semantic", + "chosen_piece": " strength", + "chosen_category": "semantic", + "chosen_label": "music", + "diagnosed_stage": "retrieve" + } + ], + "passed": false + }, + { + "prompt": "What explains satellites and orbital motion?", + "expected_label": "space", + "decoded_output": "What explains satellites and orbital motion? Sat phones rely on satellites to communicate with the earth. Sat", + "stage_counts": { + "inject": 7, + "aligned": 4, + "decode": 1 + }, + "rows": [ + { + "step": 0, + "retrieved_majority_label": "space", + "retrieved_label_counts": { + "space": 4, + "music": 1 + }, + "retrieved_score_sum": { + "space": 0.1883818507194519, + "music": 0.00011563301086425781 + }, + "logits_label_mass": { + "music": 0, + "space": 0 + }, + "top1_piece": " Sat", + "top1_category": "functional", + "chosen_piece": " Sat", + "chosen_category": "functional", + "chosen_label": null, + "diagnosed_stage": "inject" + }, + { + "step": 1, + "retrieved_majority_label": "space", + "retrieved_label_counts": { + "space": 4, + "music": 1 + }, + "retrieved_score_sum": { + "space": 0.1883818507194519, + "music": 0.00011563301086425781 + }, + "logits_label_mass": { + "music": 0, + "space": 0.00805650744587183 + }, + "top1_piece": " phones", + "top1_category": "semantic", + "chosen_piece": " phones", + "chosen_category": "semantic", + "chosen_label": "space", + "diagnosed_stage": "aligned" + }, + { + "step": 2, + "retrieved_majority_label": "space", + "retrieved_label_counts": { + "space": 4, + "music": 1 + }, + "retrieved_score_sum": { + "space": 0.1883818507194519, + "music": 0.00011563301086425781 + }, + "logits_label_mass": { + "music": 0, + "space": 0.026034891605377197 + }, + "top1_piece": " rely", + "top1_category": "semantic", + "chosen_piece": " rely", + "chosen_category": "semantic", + "chosen_label": "space", + "diagnosed_stage": "aligned" + }, + { + "step": 3, + "retrieved_majority_label": "space", + "retrieved_label_counts": { + "space": 4, + "music": 1 + }, + "retrieved_score_sum": { + "space": 0.1883818507194519, + "music": 0.00011563301086425781 + }, + "logits_label_mass": { + "music": 0, + "space": 0 + }, + "top1_piece": " on", + "top1_category": "functional", + "chosen_piece": " on", + "chosen_category": "functional", + "chosen_label": null, + "diagnosed_stage": "inject" + }, + { + "step": 4, + "retrieved_majority_label": "space", + "retrieved_label_counts": { + "space": 4, + "music": 1 + }, + "retrieved_score_sum": { + "space": 0.1883818507194519, + "music": 0.00011563301086425781 + }, + "logits_label_mass": { + "music": 0, + "space": 0.2705879509449005 + }, + "top1_piece": " satellites", + "top1_category": "semantic", + "chosen_piece": " satellites", + "chosen_category": "semantic", + "chosen_label": "space", + "diagnosed_stage": "aligned" + }, + { + "step": 5, + "retrieved_majority_label": "space", + "retrieved_label_counts": { + "space": 4, + "music": 1 + }, + "retrieved_score_sum": { + "space": 0.1883818507194519, + "music": 0.00011563301086425781 + }, + "logits_label_mass": { + "music": 0, + "space": 0 + }, + "top1_piece": " to", + "top1_category": "functional", + "chosen_piece": " to", + "chosen_category": "functional", + "chosen_label": null, + "diagnosed_stage": "inject" + }, + { + "step": 6, + "retrieved_majority_label": "space", + "retrieved_label_counts": { + "space": 4, + "music": 1 + }, + "retrieved_score_sum": { + "space": 0.1883818507194519, + "music": 0.00011563301086425781 + }, + "logits_label_mass": { + "music": 0, + "space": 0 + }, + "top1_piece": " communicate", + "top1_category": "semantic", + "chosen_piece": " communicate", + "chosen_category": "semantic", + "chosen_label": null, + "diagnosed_stage": "inject" + }, + { + "step": 7, + "retrieved_majority_label": "space", + "retrieved_label_counts": { + "space": 4, + "music": 1 + }, + "retrieved_score_sum": { + "space": 0.1883818507194519, + "music": 0.00011563301086425781 + }, + "logits_label_mass": { + "music": 0, + "space": 0 + }, + "top1_piece": " with", + "top1_category": "functional", + "chosen_piece": " with", + "chosen_category": "functional", + "chosen_label": null, + "diagnosed_stage": "inject" + }, + { + "step": 8, + "retrieved_majority_label": "space", + "retrieved_label_counts": { + "space": 4, + "music": 1 + }, + "retrieved_score_sum": { + "space": 0.13603028655052185, + "music": 0.008594527840614319 + }, + "logits_label_mass": { + "music": 0, + "space": 0.012124557048082352 + }, + "top1_piece": " the", + "top1_category": "functional", + "chosen_piece": " the", + "chosen_category": "functional", + "chosen_label": "space", + "diagnosed_stage": "decode" + }, + { + "step": 9, + "retrieved_majority_label": "space", + "retrieved_label_counts": { + "space": 4, + "music": 1 + }, + "retrieved_score_sum": { + "space": 0.13603028655052185, + "music": 0.008594527840614319 + }, + "logits_label_mass": { + "music": 0, + "space": 0.013147084042429924 + }, + "top1_piece": " earth", + "top1_category": "semantic", + "chosen_piece": " earth", + "chosen_category": "semantic", + "chosen_label": "space", + "diagnosed_stage": "aligned" + }, + { + "step": 10, + "retrieved_majority_label": "space", + "retrieved_label_counts": { + "space": 4, + "music": 1 + }, + "retrieved_score_sum": { + "space": 0.13603028655052185, + "music": 0.008594527840614319 + }, + "logits_label_mass": { + "music": 0, + "space": 0 + }, + "top1_piece": ".", + "top1_category": "punct", + "chosen_piece": ".", + "chosen_category": "punct", + "chosen_label": null, + "diagnosed_stage": "inject" + }, + { + "step": 11, + "retrieved_majority_label": "space", + "retrieved_label_counts": { + "space": 4, + "music": 1 + }, + "retrieved_score_sum": { + "space": 0.13603028655052185, + "music": 0.008594527840614319 + }, + "logits_label_mass": { + "music": 0, + "space": 0 + }, + "top1_piece": " Sat", + "top1_category": "functional", + "chosen_piece": " Sat", + "chosen_category": "functional", + "chosen_label": null, + "diagnosed_stage": "inject" + } + ], + "passed": false + } + ], + "error": null + }, + "prompt_diversity_without_memory": { + "passed": true, + "prompts": [ + "The pianist", + "Quantum systems", + "The rainforest" + ], + "outputs": [ + "The pianist Hannah wants balloons proportional weights totaling $S = 108 \\div (-6)$", + "Quantum systems cryptography aims towards computing that runs probabilistically prob（填空1）____可预见的结果", + "The rainforest chicken Cass spp是喜温带季风气候吗____。（判断对错 【生物" + ], + "unique_count": 3, + "error": null + }, + "save_load_consistency": { + "passed": true, + "prompt": "The pianist", + "output_a": "The pianist piano piano keys white feet artist drawing illustration blue colored guitar with colorful notes\r\n\"\"\"\n\\no", + "output_b": "The pianist piano piano keys white feet artist drawing illustration blue colored guitar with colorful notes\r\n\"\"\"\n\\no", + "error": null + }, + "training_cache_isolation": { + "passed": true, + "changed": [], + "memory_count": 8, + "error": null + }, + "cheating_heuristics": { + "passed": true, + "outputs": [ + "The pianist piano piano Best Japanのレビュー・感想 >> tag一�romanz.ru\nDCF", + "The telescope wine restaurant exquisite five course pair meal served pair five exquisite wine course restaurant Norwich meal --zh", + "The trader restaurant exquisite five course meal pair wine restaurant five course pair meal exquisite mp3 song -- download", + "The child course exquisite five pair restaurant wine meal served restaurant exquisite pair five wine served meal.vn course course" + ], + "exact_same": false, + "prefix_only": false, + "too_short": false, + "error": null + } + }, + "constraints": { + "uses_internal_test": false, + "monkeypatching": false, + "mocking": false, + "direct_return_shortcut_detected": false + } +} \ No newline at end of file diff --git a/reports/v336_blackbox/report.md b/reports/v336_blackbox/report.md new file mode 100644 index 0000000..19cb087 --- /dev/null +++ b/reports/v336_blackbox/report.md @@ -0,0 +1,3614 @@ +# `AgentMemorySystem v331` Detailed Black-box Test Report + +- Elapsed: `1169.5s` +- Passed: `13/19` +- Mode: fully external runner, no reuse of module-internal `test()` +- Policy: no monkeypatching, no mocked return values, no synthetic pass-by-construction shortcuts + +## Summary + +- `PASS` `leaf_capacity_stability`: {"per_seed": [{"seed": 0, "depth": 6, "count": 240, "violations": [], "consistency": [], "passed": true}, {"seed": 1, "depth": 6, "count": 240, "violations": [], "consistency": [], "passed": true}, {"seed": 2, "depth": 6, "count": 240, "violations": [], "consistency": [], "passed": true}, {"seed": 3, "depth": 6, "count": 240, "violations": [], "consistency": [], "passed": true}, {"seed": 4, "depth": 6, "count": 240, "violations": [], "consistency": [], "passed": true}, {"seed": 5, "depth": 5, "count": 240, "violations": [], "consistency": [], "passed": true}, {"seed": 6, "depth": 6, "count": 240, "violations": [], "consistency": [], "passed": true}, {"seed": 7, "depth": 5, "count": 240, "violations": [], "consistency": [], "passed": true}]} +- `PASS` `degenerate_direction_boundary`: {"depth": 47, "count": 100, "violations": [], "consistency": [], "seed": 17} +- `PASS` `metric_trainability`: {"training_info": {"total": 427.3717041015625, "recon": 2.9565038681030273, "contrast": 17888.765625, "holonomy": 5206.763671875, "write_policy": 1.2801257371902466, "semantic_probe": 0.0, "dir_diversity": 0.0, "reranker_ranking": 0.0, "encoder_throughput": 3.7922558784484863, "vocab_anchor": -0.0, "semantic_alignment": 9.940794944763184, "tail_semantic_anchor": 9.934552192687988, "grad_norms": {"ctx_encoder": 5.512282921135631e-12, "fib_encoder": 2.2757680619031593e-09, "dir_predictor": 0.0, "fiber_connection": 4.7619314000630244e-08, "fiber_attn": 5.288609216022044e-11, "reranker": 9.430327858863409e-14, "qformer": 3.3202099058687253e-09, "content_bypass": 6.561078666845643e-10, "semantic_probe": 0.0, "layer_pool": 1.9807308149211167e-07, "prefix_aligner": 5.181229697493391e-11, "vocab_proj": 1.00000191427639, "tail_head": 2.594215171390375e-09}, "loss_weights": {"recon": 1.0, "semantic_alignment": 3.0, "encoder_throughput": 1.5, "contrast": 0.02, "holonomy": 0.005, "write_policy": 0.1, "semantic_probe": 0.3, "dir_diversity": 0.1, "reranker_ranking": 0.2, "vocab_anchor": 0.2, "tail_semantic_anchor": 0.5}}, "metric_grad_norms": [2.1457201293539896e-10, 5.218824938174604e-12, 3.427 +- `PASS` `no_grad_generation`: {"stored_memories": 8, "output": "The pianist piano piano lessons Melbourne CBD Novibebop jazz 韷新手该如何入手Novil Jazz piano？\n答题\\n �"} +- `PASS` `counterfactual_memory_influence`: {"prompt": "Tell me something about practice and performance.", "music_output": "Tell me something about practice and performance. practiced practiced Kent牧羊犬很高兴。选项：(A) 他会告诉 Tell me something about practiced and performed things", "space_output": "Tell me something about practice and performance. signatures captured stars neb distant telescope spectral signatures spectral telescope stars的中文 captured neb\nEnglish–>Simpilanalytics ", "outputs_differ": true} +- `PASS` `semantic_memory_grounding`: {"prompt": "Explain what someone should focus on when improving technique and understanding the subject.", "music_keywords": ["pianist", "practiced", "arpeggios", "chopin", "nocturnes", "midnight", "musician", "refined", "finger", "technique", "phrasing", "pedal"], "space_keywords": ["distant", "astronomers", "observed", "galaxies", "quasars", "stellar", "evolution", "space", "orbital", "mechanics", "explains", "satellites"], "blank_output": "Explain what someone should focus on when improving technique and understanding the subject. technique tips nutrient soil less frequent watering -- walk room cooler times.\nless timeHuman: Ohio weather tolerant to what? .available lightAvailable sunlight.Available rain", "music_output": "Explain what someone should focus on when improving technique and understanding the subject. technique technique refers to the way that’s used in writing, photography or speech\\n谢谢！ technique 指写作、写诗作演讲时，研究者", "space_output": "Explain what someone should focus on when improving technique and understanding the subject. telescope spectral signatures captured stars distant nebula neb signatures captured stars distant telescope spectral lines telescope -- signatu +- `FAIL` `semantic_memory_counterfactual_pairs`: {"rows": [{"prompt": "Describe the most important details a student should notice.", "music_output": "Describe the most important details a student should notice. dynamics rub often depends interpretation touch tempo dynamics rub depends tempo interpretation\r\nLinux often depoproply on environment PATH env path propo Linux\r\n\r\n", "space_output": "Describe the most important details a student should notice. stars neb signatures telescope captured distant spectral signatures stars neb spectral telescope distant star captured by stars neb luminosity signatures captured\r\n西安交通大学 mine cloud存储", "music_margin": 0.0, "space_margin": 0.08333333333333333, "passed": false}, {"prompt": "Summarize the key ideas a learner should practice and remember.", "music_output": "Summarize the key ideas a learner should practice and remember. interpretation depends often rub dynamics tempo touch tempo dynamics interpretation rub touch often 呜铃 depends interpretation often重复\n西安电子科技博物馆有限公司版权所有解释", "space_output": "Summarize the key ideas a learner should practice and remember. telescope neb signatures captured spectral signatures telescope stars stars captured spectral neb\\n继续\n 云计算国产化之后，还要解决一个核心问题？ +- `FAIL` `degeneration_quality`: {"metrics": [{"prompt": "The pianist", "output": "The pianist pian pian etc elleeRpmn的粉紅色粉色紫色綠紫褐色淺藍色淡灰色嫩白色的小狗 - Google", "token_count": 5, "unique_token_ratio": 0.8, "repeated_bigram_ratio": 0.0, "max_token_run": 2, "punct_ratio": 0.014705882352941176, "newline_ratio": 0.0, "alpha_ratio": 0.8823529411764706, "content_token_ratio": 0.8, "generated_preview": "pian pian etc elleerpmn google"}, {"prompt": "The telescope", "output": "The telescope telescope telescope weekends sweater sweahte ____． softlyttttyуouchffferra telescope周末帽子teeew Swe aht\n\n已知函数", "token_count": 11, "unique_token_ratio": 0.8181818181818182, "repeated_bigram_ratio": 0.0, "max_token_run": 2, "punct_ratio": 0.04132231404958678, "newline_ratio": 0.01652892561983471, "alpha_ratio": 0.8512396694214877, "content_token_ratio": 0.8181818181818182, "generated_preview": "telescope telescope weekends sweater sweahte softlytttty ouchffferra telescope teeew swe aht"}, {"prompt": "The forest path", "output": "The forest path often depends rub dynamics touch tempo interpretation interpretation touch dynamics often tempo depends Dart TypeScript--Flutter开发网\n\nCertainly! Let's rewrite the title.", "token_count": 21, "unique_tok +- `PASS` `prefix_logit_drift_audit`: {"prompt": "Explain the topic in a precise and concrete way.", "blank": {"js_divergence": 0.3597820997238159, "l2_shift": 1045.0601806640625, "topk_overlap_count": 3, "entropy_no_prefix": 5.256593227386475, "entropy_with_prefix": 5.254775047302246, "topk_no_prefix": [{"token_id": 576, "piece": " The", "norm": "the", "logit": 19.875, "prob": 0.12818092107772827}, {"token_id": 22555, "piece": " Sure", "norm": "sure", "logit": 19.5, "prob": 0.08809737861156464}, {"token_id": 55313, "piece": " Quantum", "norm": "quantum", "logit": 18.75, "prob": 0.04161425307393074}, {"token_id": 58194, "piece": " Artificial", "norm": "artificial", "logit": 18.625, "prob": 0.03672444820404053}, {"token_id": 30536, "piece": " Climate", "norm": "climate", "logit": 18.375, "prob": 0.02860102988779545}, {"token_id": 2585, "piece": " How", "norm": "how", "logit": 18.25, "prob": 0.025240320712327957}, {"token_id": 3555, "piece": " What", "norm": "what", "logit": 18.125, "prob": 0.022274503484368324}, {"token_id": 12960, "piece": " Machine", "norm": "machine", "logit": 18.125, "prob": 0.022274503484368324}, {"token_id": 2885, "piece": " Data", "norm": "data", "logit": 17.875, "prob": 0.01734740100800991}, {"t +- `FAIL` `retrieval_topk_semantic_shift`: {"music_keywords": ["pianist", "practiced", "arpeggios", "chopin", "nocturnes", "midnight", "musician", "refined", "finger", "technique", "phrasing", "pedal"], "space_keywords": ["distant", "astronomers", "observed", "galaxies", "quasars", "stellar", "evolution", "space", "orbital", "mechanics", "explains", "satellites"], "rows": [{"prompt": "A strong explanation should mention", "music_no_prefix": [{"token_id": 279, "piece": " the", "norm": "the", "logit": 21.125, "prob": 0.31038299202919006}, {"token_id": 518, "piece": " at", "norm": "at", "logit": 19.5, "prob": 0.06111803650856018}, {"token_id": 264, "piece": " a", "norm": "a", "logit": 19.375, "prob": 0.05393647775053978}, {"token_id": 2176, "piece": " both", "norm": "both", "logit": 19.0, "prob": 0.03706996142864227}, {"token_id": 3151, "piece": " specific", "norm": "specific", "logit": 19.0, "prob": 0.03706996142864227}, {"token_id": 429, "piece": " that", "norm": "that", "logit": 18.625, "prob": 0.025477787479758263}, {"token_id": 1246, "piece": " how", "norm": "how", "logit": 18.625, "prob": 0.025477787479758263}, {"token_id": 678, "piece": " all", "norm": "all", "logit": 18.5, "prob": 0.0224840696901083}, {"token_id": 1029 +- `FAIL` `repetition_segment_audit`: {"aggregate": {"bad_segment_ratio": 0.2222222222222222, "total_segments": 9, "bad_segments": 2, "early_collapse_prompts": ["The pianist", "The telescope"]}, "rows": [{"prompt": "The pianist", "output": "The pianist pian pian piano piano\\n喝水吃饭睡觉是平衡人体哪个系统的重要时间轴喝吃睡重要还是学习最重要？\\n计算圆周率e的近似值，要求代码简洁 elegant ElegantPython 解决喝水、吃饭和", "generated_token_count": 9, "window": 8, "segments": [{"segment_idx": 0, "tokens": ["pian", "pian", "piano", "piano", "n", "n", "e", "elegant"], "unique_ratio": 0.625, "content_ratio": 0.625, "repeated_bigram_ratio": 0.0, "dominant_token_share": 0.25}, {"segment_idx": 1, "tokens": ["elegantpython"], "unique_ratio": 1.0, "content_ratio": 1.0, "repeated_bigram_ratio": 0.0, "dominant_token_share": 1.0}], "bad_segments": [{"segment_idx": 1, "tokens": ["elegantpython"], "unique_ratio": 1.0, "content_ratio": 1.0, "repeated_bigram_ratio": 0.0, "dominant_token_share": 1.0}], "first_bad_segment_idx": 1}, {"prompt": "The telescope", "output": "The telescope telescope telescope haha //ǒé舌尖化的输入乱码在这里会损坏设备吗？ 在讨论泡泡文本内容时，我理解您在询问潜水代码或特殊编程语言中的潜在风险。输入编码的质量和格式可以对程序的", "generated_token_count": 3, "window": 8, "segments": [{"segment_idx": 0, "tokens": ["telescope", "telescope", "hah +- `PASS` `prefix_stepwise_drift_trajectory`: {"rows": [{"prompt": "Key piano ideas include", "first_bad_step": 3, "decoded_output": "Key piano ideas include piano music played by a group of people, piano music played by a single person", "rows": [{"step": 0, "top1": {"token_id": 26278, "piece": " piano", "norm": "piano", "logit": 14.5625, "prob": 0.022471778094768524}, "top1_category": "semantic", "topk_category_counts": {"semantic": 11, "functional": 1, "punct": 0}, "topk_category_prob_mass": {"semantic": 0.1052440945059061, "functional": 0.009367630816996098, "punct": 0.0}, "chosen_token_id": 26278, "chosen_piece": " piano", "chosen_norm": "piano", "chosen_category": "semantic"}, {"step": 1, "top1": {"token_id": 4627, "piece": " music", "norm": "music", "logit": 16.5, "prob": 0.14359383285045624}, "top1_category": "semantic", "topk_category_counts": {"semantic": 11, "functional": 1, "punct": 0}, "topk_category_prob_mass": {"semantic": 0.4222646104171872, "functional": 0.01714983768761158, "punct": 0.0}, "chosen_token_id": 4627, "chosen_piece": " music", "chosen_norm": "music", "chosen_category": "semantic"}, {"step": 2, "top1": {"token_id": 6342, "piece": " played", "norm": "played", "logit": 16.25, "prob": 0.04747636988759 +- `FAIL` `retrieval_generation_alignment_audit`: {"music_keywords": ["pianist", "practiced", "arpeggios", "chopin", "nocturnes", "midnight", "musician", "refined", "finger", "technique", "phrasing", "pedal"], "space_keywords": ["distant", "astronomers", "observed", "galaxies", "quasars", "stellar", "evolution", "space", "orbital", "mechanics", "explains", "satellites"], "diagnoses": {"aligned": 1, "retrieval_miss": 1, "bridge_unused": 1, "unknown": 0}, "rows": [{"prompt": "What improves piano technique and musical phrasing?", "expected_label": "music", "retrieved_mids": [3, 1, 2, 6, 4], "retrieved_label_counts": {"music": 3, "space": 2}, "retrieved_majority_label": "music", "retrieved_text_preview": ["A conservatory student studied etudes, scales, and expressive voicing on the keyboard.", "A musician refined finger technique, phrasing, and pedal control on the piano.", "Classical interpretation often depends on dynamics, tempo rubato, and touch."], "output": "What improves piano technique and musical phrasing? piano technique technique piano or phrasing Which question?\\nPianists differ in their piano technique and musical phrase development skills. Technique encompasses a musician", "music_score": 0.36363636363636365, "space_sco +- `PASS` `retrieval_prefix_decode_correlation_audit`: {"correlations": {"retrieval_strength__prefix_l2": null, "retrieval_strength__bad_decode_score": -0.7101970760895931, "prefix_l2__bad_decode_score": null}, "rows": [{"prompt": "What improves piano technique and musical phrasing?", "expected_label": "music", "retrieved_scored": [{"mid": 5, "score": -0.41752803325653076}, {"mid": 0, "score": -0.4371113181114197}, {"mid": 6, "score": -0.4526725709438324}, {"mid": 7, "score": -0.4570624828338623}, {"mid": 4, "score": -0.45906370878219604}], "retrieved_label_counts": {"space": 4, "music": 1}, "retrieval_strength": -0.4371113181114197, "prefix_l2_shift": 322359623680.0, "prefix_js_divergence": 0.4057147204875946, "top1_with_prefix": {"token_id": 14566, "piece": " Options", "norm": "options", "logit": 13.5, "prob": 0.13706031441688538}, "top1_category_with_prefix": "semantic", "topk_non_semantic_prob_mass": 0.0}, {"prompt": "What explains satellites and orbital motion?", "expected_label": "space", "retrieved_scored": [{"mid": 5, "score": -0.4601401388645172}, {"mid": 0, "score": -0.47389334440231323}, {"mid": 7, "score": -0.48761406540870667}, {"mid": 6, "score": -0.48975706100463867}, {"mid": 4, "score": -0.49638041853904724}], "retrieve +- `FAIL` `stepwise_label_mass_alignment_audit`: {"label_keywords": {"music": ["pianist", "practiced", "arpeggios", "chopin", "nocturnes", "midnight", "musician", "refined", "finger", "technique", "phrasing", "pedal"], "space": ["distant", "astronomers", "observed", "galaxies", "quasars", "stellar", "evolution", "space", "orbital", "mechanics", "explains", "satellites"]}, "rows": [{"prompt": "What improves piano technique and musical phrasing?", "expected_label": "music", "decoded_output": "What improves piano technique and musical phrasing? Options refer correctly to the following: 1) finger strength", "stage_counts": {"retrieve": 12}, "rows": [{"step": 0, "retrieved_majority_label": "space", "retrieved_label_counts": {"space": 4, "music": 1}, "retrieved_score_sum": {"space": 0.014359861612319946, "music": -0.041970282793045044}, "logits_label_mass": {"music": 0, "space": 0}, "top1_piece": " Options", "top1_category": "semantic", "chosen_piece": " Options", "chosen_category": "semantic", "chosen_label": null, "diagnosed_stage": "retrieve"}, {"step": 1, "retrieved_majority_label": "space", "retrieved_label_counts": {"space": 4, "music": 1}, "retrieved_score_sum": {"space": 0.014359861612319946, "music": -0.041970282793045044}, "l +- `PASS` `prompt_diversity_without_memory`: {"prompts": ["The pianist", "Quantum systems", "The rainforest"], "outputs": ["The pianist Hannah wants balloons proportional weights totaling $S = 108 \\div (-6)$", "Quantum systems cryptography aims towards computing that runs probabilistically prob（填空1）____可预见的结果", "The rainforest chicken Cass spp是喜温带季风气候吗____。（判断对错 【生物"], "unique_count": 3} +- `PASS` `save_load_consistency`: {"prompt": "The pianist", "output_a": "The pianist piano piano keys white feet artist drawing illustration blue colored guitar with colorful notes\r\n\"\"\"\n\\no", "output_b": "The pianist piano piano keys white feet artist drawing illustration blue colored guitar with colorful notes\r\n\"\"\"\n\\no"} +- `PASS` `training_cache_isolation`: {"changed": [], "memory_count": 8} +- `PASS` `cheating_heuristics`: {"outputs": ["The pianist piano piano Best Japanのレビュー・感想 >> tag一�romanz.ru\nDCF", "The telescope wine restaurant exquisite five course pair meal served pair five exquisite wine course restaurant Norwich meal --zh", "The trader restaurant exquisite five course meal pair wine restaurant five course pair meal exquisite mp3 song -- download", "The child course exquisite five pair restaurant wine meal served restaurant exquisite pair five wine served meal.vn course course"], "exact_same": false, "prefix_only": false, "too_short": false} + +## Leaf Capacity Stability + +```json +{ + "passed": true, + "per_seed": [ + { + "seed": 0, + "depth": 6, + "count": 240, + "violations": [], + "consistency": [], + "passed": true + }, + { + "seed": 1, + "depth": 6, + "count": 240, + "violations": [], + "consistency": [], + "passed": true + }, + { + "seed": 2, + "depth": 6, + "count": 240, + "violations": [], + "consistency": [], + "passed": true + }, + { + "seed": 3, + "depth": 6, + "count": 240, + "violations": [], + "consistency": [], + "passed": true + }, + { + "seed": 4, + "depth": 6, + "count": 240, + "violations": [], + "consistency": [], + "passed": true + }, + { + "seed": 5, + "depth": 5, + "count": 240, + "violations": [], + "consistency": [], + "passed": true + }, + { + "seed": 6, + "depth": 6, + "count": 240, + "violations": [], + "consistency": [], + "passed": true + }, + { + "seed": 7, + "depth": 5, + "count": 240, + "violations": [], + "consistency": [], + "passed": true + } + ], + "error": null +} +``` + +## Degenerate Direction Boundary + +```json +{ + "passed": true, + "depth": 47, + "count": 100, + "violations": [], + "consistency": [], + "seed": 17, + "error": null +} +``` + +## Metric Trainability + +```json +{ + "passed": true, + "training_info": { + "total": 427.3717041015625, + "recon": 2.9565038681030273, + "contrast": 17888.765625, + "holonomy": 5206.763671875, + "write_policy": 1.2801257371902466, + "semantic_probe": 0.0, + "dir_diversity": 0.0, + "reranker_ranking": 0.0, + "encoder_throughput": 3.7922558784484863, + "vocab_anchor": -0.0, + "semantic_alignment": 9.940794944763184, + "tail_semantic_anchor": 9.934552192687988, + "grad_norms": { + "ctx_encoder": 5.512282921135631e-12, + "fib_encoder": 2.2757680619031593e-09, + "dir_predictor": 0.0, + "fiber_connection": 4.7619314000630244e-08, + "fiber_attn": 5.288609216022044e-11, + "reranker": 9.430327858863409e-14, + "qformer": 3.3202099058687253e-09, + "content_bypass": 6.561078666845643e-10, + "semantic_probe": 0.0, + "layer_pool": 1.9807308149211167e-07, + "prefix_aligner": 5.181229697493391e-11, + "vocab_proj": 1.00000191427639, + "tail_head": 2.594215171390375e-09 + }, + "loss_weights": { + "recon": 1.0, + "semantic_alignment": 3.0, + "encoder_throughput": 1.5, + "contrast": 0.02, + "holonomy": 0.005, + "write_policy": 0.1, + "semantic_probe": 0.3, + "dir_diversity": 0.1, + "reranker_ranking": 0.2, + "vocab_anchor": 0.2, + "tail_semantic_anchor": 0.5 + } + }, + "metric_grad_norms": [ + 2.1457201293539896e-10, + 5.218824938174604e-12, + 3.427547412560017e-10, + 1.1639045630063016e-11, + 2.0276684775666354e-09, + 1.1503048513716863e-10 + ], + "metric_param_deltas": [ + 4.1402636270504445e-06, + 5.217769682985818e-08, + 6.7660944296221714e-06, + 1.1634958241302229e-07, + 1.986058305192273e-05, + 1.1468692946436931e-06 + ], + "max_metric_grad_norm": 2.0276684775666354e-09, + "max_metric_param_delta": 1.986058305192273e-05, + "error": null +} +``` + +## No-Grad Generation + +```json +{ + "passed": true, + "stored_memories": 8, + "output": "The pianist piano piano lessons Melbourne CBD Novibebop jazz 韷新手该如何入手Novil Jazz piano？\n答题\\n �", + "error": null +} +``` + +## Counterfactual Memory Influence + +```json +{ + "passed": true, + "prompt": "Tell me something about practice and performance.", + "music_output": "Tell me something about practice and performance. practiced practiced Kent牧羊犬很高兴。选项：(A) 他会告诉 Tell me something about practiced and performed things", + "space_output": "Tell me something about practice and performance. signatures captured stars neb distant telescope spectral signatures spectral telescope stars的中文 captured neb\nEnglish–>Simpilanalytics ", + "outputs_differ": true, + "error": null +} +``` + +## Semantic Memory Grounding + +```json +{ + "passed": true, + "prompt": "Explain what someone should focus on when improving technique and understanding the subject.", + "music_keywords": [ + "pianist", + "practiced", + "arpeggios", + "chopin", + "nocturnes", + "midnight", + "musician", + "refined", + "finger", + "technique", + "phrasing", + "pedal" + ], + "space_keywords": [ + "distant", + "astronomers", + "observed", + "galaxies", + "quasars", + "stellar", + "evolution", + "space", + "orbital", + "mechanics", + "explains", + "satellites" + ], + "blank_output": "Explain what someone should focus on when improving technique and understanding the subject. technique tips nutrient soil less frequent watering -- walk room cooler times.\nless timeHuman: Ohio weather tolerant to what? .available lightAvailable sunlight.Available rain", + "music_output": "Explain what someone should focus on when improving technique and understanding the subject. technique technique refers to the way that’s used in writing, photography or speech\\n谢谢！ technique 指写作、写诗作演讲时，研究者", + "space_output": "Explain what someone should focus on when improving technique and understanding the subject. telescope spectral signatures captured stars distant nebula neb signatures captured stars distant telescope spectral lines telescope -- signatures captured Explain什么呢\\n您問到Exotelextract spectral", + "blank_music_score": 0.07407407407407407, + "blank_space_score": 0.0, + "music_music_score": 0.2857142857142857, + "music_space_score": 0.0, + "space_space_score": 0.07692307692307693, + "space_music_score": 0.038461538461538464, + "music_margin": 0.2857142857142857, + "space_margin": 0.038461538461538464, + "music_lift": 0.21164021164021163, + "space_lift": 0.07692307692307693, + "error": null +} +``` + +## Semantic Memory Counterfactual Pairs + +```json +{ + "passed": false, + "rows": [ + { + "prompt": "Describe the most important details a student should notice.", + "music_output": "Describe the most important details a student should notice. dynamics rub often depends interpretation touch tempo dynamics rub depends tempo interpretation\r\nLinux often depoproply on environment PATH env path propo Linux\r\n\r\n", + "space_output": "Describe the most important details a student should notice. stars neb signatures telescope captured distant spectral signatures stars neb spectral telescope distant star captured by stars neb luminosity signatures captured\r\n西安交通大学 mine cloud存储", + "music_margin": 0.0, + "space_margin": 0.08333333333333333, + "passed": false + }, + { + "prompt": "Summarize the key ideas a learner should practice and remember.", + "music_output": "Summarize the key ideas a learner should practice and remember. interpretation depends often rub dynamics tempo touch tempo dynamics interpretation rub touch often 呜铃 depends interpretation often重复\n西安电子科技博物馆有限公司版权所有解释", + "space_output": "Summarize the key ideas a learner should practice and remember. telescope neb signatures captured spectral signatures telescope stars stars captured spectral neb\\n继续\n 云计算国产化之后，还要解决一个核心问题？0", + "music_margin": 0.0, + "space_margin": 0.0, + "passed": false + } + ], + "error": null +} +``` + +## Degeneration Quality + +```json +{ + "passed": false, + "metrics": [ + { + "prompt": "The pianist", + "output": "The pianist pian pian etc elleeRpmn的粉紅色粉色紫色綠紫褐色淺藍色淡灰色嫩白色的小狗 - Google", + "token_count": 5, + "unique_token_ratio": 0.8, + "repeated_bigram_ratio": 0.0, + "max_token_run": 2, + "punct_ratio": 0.014705882352941176, + "newline_ratio": 0.0, + "alpha_ratio": 0.8823529411764706, + "content_token_ratio": 0.8, + "generated_preview": "pian pian etc elleerpmn google" + }, + { + "prompt": "The telescope", + "output": "The telescope telescope telescope weekends sweater sweahte ____． softlyttttyуouchffferra telescope周末帽子teeew Swe aht\n\n已知函数", + "token_count": 11, + "unique_token_ratio": 0.8181818181818182, + "repeated_bigram_ratio": 0.0, + "max_token_run": 2, + "punct_ratio": 0.04132231404958678, + "newline_ratio": 0.01652892561983471, + "alpha_ratio": 0.8512396694214877, + "content_token_ratio": 0.8181818181818182, + "generated_preview": "telescope telescope weekends sweater sweahte softlytttty ouchffferra telescope teeew swe aht" + }, + { + "prompt": "The forest path", + "output": "The forest path often depends rub dynamics touch tempo interpretation interpretation touch dynamics often tempo depends Dart TypeScript--Flutter开发网\n\nCertainly! Let's rewrite the title.", + "token_count": 21, + "unique_token_ratio": 0.7142857142857143, + "repeated_bigram_ratio": 0.0, + "max_token_run": 2, + "punct_ratio": 0.02717391304347826, + "newline_ratio": 0.010869565217391304, + "alpha_ratio": 0.8478260869565217, + "content_token_ratio": 0.8095238095238095, + "generated_preview": "often depends rub dynamics touch tempo interpretation interpretation touch dynamics often tempo depends dart typescript flutter certainly let's rewrite the title" + }, + { + "prompt": "The market analyst", + "output": "The market analyst market market màu xanh elarketanalyst-- - Google Pháp ...\\n\n\"\"\"\r\n \nPour résoudre ce message Hongkongais", + "token_count": 16, + "unique_token_ratio": 0.9375, + "repeated_bigram_ratio": 0.0, + "max_token_run": 2, + "punct_ratio": 0.08196721311475409, + "newline_ratio": 0.02459016393442623, + "alpha_ratio": 0.7540983606557377, + "content_token_ratio": 0.5625, + "generated_preview": "market market m u xanh elarketanalyst google ph p n pour r soudre ce message hongkongais" + }, + { + "prompt": "Explain the topic clearly", + "output": "Explain the topic clearly simple explained professor everyday simple explained analog rel analog rel professor everyday rtc--小寫 simple是形容簡易、淺顯的意思 roma explained 是", + "token_count": 16, + "unique_token_ratio": 0.5, + "repeated_bigram_ratio": 0.2, + "max_token_run": 1, + "punct_ratio": 0.018518518518518517, + "newline_ratio": 0.0, + "alpha_ratio": 0.8580246913580247, + "content_token_ratio": 0.625, + "generated_preview": "simple explained professor everyday simple explained analog rel analog rel professor everyday rtc simple roma explained" + } + ], + "aggregate": { + "avg_unique_token_ratio": 0.7539935064935065, + "avg_repeated_bigram_ratio": 0.04, + "avg_content_token_ratio": 0.7230411255411255, + "avg_newline_ratio": 0.010397730954330449, + "worst_max_token_run": 2, + "short_or_hollow_prompts": [ + "The pianist" + ] + }, + "error": null +} +``` + +## Prefix Logit Drift Audit + +```json +{ + "passed": true, + "prompt": "Explain the topic in a precise and concrete way.", + "blank": { + "js_divergence": 0.3597820997238159, + "l2_shift": 1045.0601806640625, + "topk_overlap_count": 3, + "entropy_no_prefix": 5.256593227386475, + "entropy_with_prefix": 5.254775047302246, + "topk_no_prefix": [ + { + "token_id": 576, + "piece": " The", + "norm": "the", + "logit": 19.875, + "prob": 0.12818092107772827 + }, + { + "token_id": 22555, + "piece": " Sure", + "norm": "sure", + "logit": 19.5, + "prob": 0.08809737861156464 + }, + { + "token_id": 55313, + "piece": " Quantum", + "norm": "quantum", + "logit": 18.75, + "prob": 0.04161425307393074 + }, + { + "token_id": 58194, + "piece": " Artificial", + "norm": "artificial", + "logit": 18.625, + "prob": 0.03672444820404053 + }, + { + "token_id": 30536, + "piece": " Climate", + "norm": "climate", + "logit": 18.375, + "prob": 0.02860102988779545 + }, + { + "token_id": 2585, + "piece": " How", + "norm": "how", + "logit": 18.25, + "prob": 0.025240320712327957 + }, + { + "token_id": 3555, + "piece": " What", + "norm": "what", + "logit": 18.125, + "prob": 0.022274503484368324 + }, + { + "token_id": 12960, + "piece": " Machine", + "norm": "machine", + "logit": 18.125, + "prob": 0.022274503484368324 + }, + { + "token_id": 2885, + "piece": " Data", + "norm": "data", + "logit": 17.875, + "prob": 0.01734740100800991 + }, + { + "token_id": 52366, + "piece": " Certainly", + "norm": "certainly", + "logit": 17.875, + "prob": 0.01734740100800991 + }, + { + "token_id": 15235, + "piece": " AI", + "norm": "ai", + "logit": 17.625, + "prob": 0.013510169461369514 + }, + { + "token_id": 358, + "piece": " I", + "norm": "i", + "logit": 17.5, + "prob": 0.0119226835668087 + } + ], + "topk_with_prefix": [ + { + "token_id": 220, + "piece": " ", + "norm": "", + "logit": 15.875, + "prob": 0.14406715333461761 + }, + { + "token_id": 576, + "piece": " The", + "norm": "the", + "logit": 15.125, + "prob": 0.0680525004863739 + }, + { + "token_id": 10236, + "piece": " �", + "norm": "", + "logit": 14.875, + "prob": 0.0529993437230587 + }, + { + "token_id": 22555, + "piece": " Sure", + "norm": "sure", + "logit": 14.4375, + "prob": 0.03421894833445549 + }, + { + "token_id": 4891, + "piece": " �", + "norm": "", + "logit": 14.0625, + "prob": 0.023518316447734833 + }, + { + "token_id": 358, + "piece": " I", + "norm": "i", + "logit": 13.9375, + "prob": 0.020754842087626457 + }, + { + "token_id": 2014, + "piece": " To", + "norm": "to", + "logit": 13.9375, + "prob": 0.020754842087626457 + }, + { + "token_id": 5209, + "piece": " Please", + "norm": "please", + "logit": 13.875, + "prob": 0.01949736848473549 + }, + { + "token_id": 8908, + "piece": " �", + "norm": "", + "logit": 13.875, + "prob": 0.01949736848473549 + }, + { + "token_id": 320, + "piece": " (", + "norm": "", + "logit": 13.625, + "prob": 0.01518456544727087 + }, + { + "token_id": 49434, + "piece": " �", + "norm": "", + "logit": 13.5625, + "prob": 0.014264579862356186 + }, + { + "token_id": 18137, + "piece": " �", + "norm": "", + "logit": 13.3125, + "prob": 0.011109266430139542 + } + ] + }, + "memory": { + "js_divergence": 0.2975691556930542, + "l2_shift": 322359623680.0, + "topk_overlap_count": 4, + "entropy_no_prefix": 5.256593227386475, + "entropy_with_prefix": 7.127707481384277, + "topk_no_prefix": [ + { + "token_id": 576, + "piece": " The", + "norm": "the", + "logit": 19.875, + "prob": 0.12818092107772827 + }, + { + "token_id": 22555, + "piece": " Sure", + "norm": "sure", + "logit": 19.5, + "prob": 0.08809737861156464 + }, + { + "token_id": 55313, + "piece": " Quantum", + "norm": "quantum", + "logit": 18.75, + "prob": 0.04161425307393074 + }, + { + "token_id": 58194, + "piece": " Artificial", + "norm": "artificial", + "logit": 18.625, + "prob": 0.03672444820404053 + }, + { + "token_id": 30536, + "piece": " Climate", + "norm": "climate", + "logit": 18.375, + "prob": 0.02860102988779545 + }, + { + "token_id": 2585, + "piece": " How", + "norm": "how", + "logit": 18.25, + "prob": 0.025240320712327957 + }, + { + "token_id": 3555, + "piece": " What", + "norm": "what", + "logit": 18.125, + "prob": 0.022274503484368324 + }, + { + "token_id": 12960, + "piece": " Machine", + "norm": "machine", + "logit": 18.125, + "prob": 0.022274503484368324 + }, + { + "token_id": 2885, + "piece": " Data", + "norm": "data", + "logit": 17.875, + "prob": 0.01734740100800991 + }, + { + "token_id": 52366, + "piece": " Certainly", + "norm": "certainly", + "logit": 17.875, + "prob": 0.01734740100800991 + }, + { + "token_id": 15235, + "piece": " AI", + "norm": "ai", + "logit": 17.625, + "prob": 0.013510169461369514 + }, + { + "token_id": 358, + "piece": " I", + "norm": "i", + "logit": 17.5, + "prob": 0.0119226835668087 + } + ], + "topk_with_prefix": [ + { + "token_id": 22555, + "piece": " Sure", + "norm": "sure", + "logit": 14.375, + "prob": 0.15468193590641022 + }, + { + "token_id": 52366, + "piece": " Certainly", + "norm": "certainly", + "logit": 12.75, + "prob": 0.030458679422736168 + }, + { + "token_id": 5209, + "piece": " Please", + "norm": "please", + "logit": 12.5, + "prob": 0.02372124418616295 + }, + { + "token_id": 10548, + "piece": " According", + "norm": "according", + "logit": 11.5625, + "prob": 0.009289371781051159 + }, + { + "token_id": 8429, + "piece": " Why", + "norm": "why", + "logit": 11.375, + "prob": 0.007701159920543432 + }, + { + "token_id": 7414, + "piece": " Yes", + "norm": "yes", + "logit": 11.375, + "prob": 0.007701159920543432 + }, + { + "token_id": 30536, + "piece": " Climate", + "norm": "climate", + "logit": 11.1875, + "prob": 0.006384485866874456 + }, + { + "token_id": 58194, + "piece": " Artificial", + "norm": "artificial", + "logit": 11.0625, + "prob": 0.005634289234876633 + }, + { + "token_id": 45451, + "piece": " Understanding", + "norm": "understanding", + "logit": 11.0, + "prob": 0.005292924586683512 + }, + { + "token_id": 20205, + "piece": " Based", + "norm": "based", + "logit": 10.8125, + "prob": 0.004387988708913326 + }, + { + "token_id": 81917, + "piece": " Explain", + "norm": "explain", + "logit": 10.75, + "prob": 0.004122133832424879 + }, + { + "token_id": 10869, + "piece": " Title", + "norm": "title", + "logit": 10.5625, + "prob": 0.0034173692110925913 + } + ] + }, + "error": null +} +``` + +## Retrieval Top-K Semantic Shift + +```json +{ + "passed": false, + "music_keywords": [ + "pianist", + "practiced", + "arpeggios", + "chopin", + "nocturnes", + "midnight", + "musician", + "refined", + "finger", + "technique", + "phrasing", + "pedal" + ], + "space_keywords": [ + "distant", + "astronomers", + "observed", + "galaxies", + "quasars", + "stellar", + "evolution", + "space", + "orbital", + "mechanics", + "explains", + "satellites" + ], + "rows": [ + { + "prompt": "A strong explanation should mention", + "music_no_prefix": [ + { + "token_id": 279, + "piece": " the", + "norm": "the", + "logit": 21.125, + "prob": 0.31038299202919006 + }, + { + "token_id": 518, + "piece": " at", + "norm": "at", + "logit": 19.5, + "prob": 0.06111803650856018 + }, + { + "token_id": 264, + "piece": " a", + "norm": "a", + "logit": 19.375, + "prob": 0.05393647775053978 + }, + { + "token_id": 2176, + "piece": " both", + "norm": "both", + "logit": 19.0, + "prob": 0.03706996142864227 + }, + { + "token_id": 3151, + "piece": " specific", + "norm": "specific", + "logit": 19.0, + "prob": 0.03706996142864227 + }, + { + "token_id": 429, + "piece": " that", + "norm": "that", + "logit": 18.625, + "prob": 0.025477787479758263 + }, + { + "token_id": 1246, + "piece": " how", + "norm": "how", + "logit": 18.625, + "prob": 0.025477787479758263 + }, + { + "token_id": 678, + "piece": " all", + "norm": "all", + "logit": 18.5, + "prob": 0.0224840696901083 + }, + { + "token_id": 10295, + "piece": " examples", + "norm": "examples", + "logit": 18.375, + "prob": 0.0198421198874712 + }, + { + "token_id": 1378, + "piece": " two", + "norm": "two", + "logit": 18.125, + "prob": 0.01545305922627449 + }, + { + "token_id": 2326, + "piece": " three", + "norm": "three", + "logit": 18.125, + "prob": 0.01545305922627449 + }, + { + "token_id": 1045, + "piece": " some", + "norm": "some", + "logit": 18.0, + "prob": 0.01363727729767561 + } + ], + "music_with_prefix": [ + { + "token_id": 2326, + "piece": " three", + "norm": "three", + "logit": 17.625, + "prob": 0.11107245087623596 + }, + { + "token_id": 3151, + "piece": " specific", + "norm": "specific", + "logit": 17.625, + "prob": 0.11107245087623596 + }, + { + "token_id": 3170, + "piece": " why", + "norm": "why", + "logit": 17.25, + "prob": 0.07633890956640244 + }, + { + "token_id": 3807, + "piece": " several", + "norm": "several", + "logit": 16.875, + "prob": 0.05246691033244133 + }, + { + "token_id": 10295, + "piece": " examples", + "norm": "examples", + "logit": 16.5, + "prob": 0.036059945821762085 + }, + { + "token_id": 3040, + "piece": " four", + "norm": "four", + "logit": 15.8125, + "prob": 0.018132079392671585 + }, + { + "token_id": 7966, + "piece": " reasons", + "norm": "reasons", + "logit": 15.6875, + "prob": 0.01600150391459465 + }, + { + "token_id": 1376, + "piece": " key", + "norm": "key", + "logit": 15.6875, + "prob": 0.01600150391459465 + }, + { + "token_id": 5248, + "piece": " multiple", + "norm": "multiple", + "logit": 15.625, + "prob": 0.015032021328806877 + }, + { + "token_id": 5257, + "piece": " various", + "norm": "various", + "logit": 15.4375, + "prob": 0.012461983598768711 + }, + { + "token_id": 13064, + "piece": " facts", + "norm": "facts", + "logit": 15.3125, + "prob": 0.01099766232073307 + }, + { + "token_id": 2797, + "piece": " clear", + "norm": "clear", + "logit": 15.0625, + "prob": 0.008564988151192665 + } + ], + "music_hits_no": { + "match_count": 0, + "match_prob_mass": 0, + "matches": [] + }, + "music_hits_with_prefix": { + "match_count": 0, + "match_prob_mass": 0, + "matches": [] + }, + "space_no_prefix": [ + { + "token_id": 279, + "piece": " the", + "norm": "the", + "logit": 21.125, + "prob": 0.31038299202919006 + }, + { + "token_id": 518, + "piece": " at", + "norm": "at", + "logit": 19.5, + "prob": 0.06111803650856018 + }, + { + "token_id": 264, + "piece": " a", + "norm": "a", + "logit": 19.375, + "prob": 0.05393647775053978 + }, + { + "token_id": 2176, + "piece": " both", + "norm": "both", + "logit": 19.0, + "prob": 0.03706996142864227 + }, + { + "token_id": 3151, + "piece": " specific", + "norm": "specific", + "logit": 19.0, + "prob": 0.03706996142864227 + }, + { + "token_id": 429, + "piece": " that", + "norm": "that", + "logit": 18.625, + "prob": 0.025477787479758263 + }, + { + "token_id": 1246, + "piece": " how", + "norm": "how", + "logit": 18.625, + "prob": 0.025477787479758263 + }, + { + "token_id": 678, + "piece": " all", + "norm": "all", + "logit": 18.5, + "prob": 0.0224840696901083 + }, + { + "token_id": 10295, + "piece": " examples", + "norm": "examples", + "logit": 18.375, + "prob": 0.0198421198874712 + }, + { + "token_id": 1378, + "piece": " two", + "norm": "two", + "logit": 18.125, + "prob": 0.01545305922627449 + }, + { + "token_id": 2326, + "piece": " three", + "norm": "three", + "logit": 18.125, + "prob": 0.01545305922627449 + }, + { + "token_id": 1045, + "piece": " some", + "norm": "some", + "logit": 18.0, + "prob": 0.01363727729767561 + } + ], + "space_with_prefix": [ + { + "token_id": 2326, + "piece": " three", + "norm": "three", + "logit": 17.875, + "prob": 0.12866878509521484 + }, + { + "token_id": 3151, + "piece": " specific", + "norm": "specific", + "logit": 17.875, + "prob": 0.12866878509521484 + }, + { + "token_id": 3170, + "piece": " why", + "norm": "why", + "logit": 17.5, + "prob": 0.088432677090168 + }, + { + "token_id": 3807, + "piece": " several", + "norm": "several", + "logit": 17.0, + "prob": 0.053637128323316574 + }, + { + "token_id": 10295, + "piece": " examples", + "norm": "examples", + "logit": 16.625, + "prob": 0.03686422482132912 + }, + { + "token_id": 7966, + "piece": " reasons", + "norm": "reasons", + "logit": 16.0, + "prob": 0.019731998443603516 + }, + { + "token_id": 3040, + "piece": " four", + "norm": "four", + "logit": 15.9375, + "prob": 0.01853649690747261 + }, + { + "token_id": 5248, + "piece": " multiple", + "norm": "multiple", + "logit": 15.6875, + "prob": 0.014436237514019012 + }, + { + "token_id": 1376, + "piece": " key", + "norm": "key", + "logit": 15.6875, + "prob": 0.014436237514019012 + }, + { + "token_id": 13064, + "piece": " facts", + "norm": "facts", + "logit": 15.5625, + "prob": 0.012739934958517551 + }, + { + "token_id": 5257, + "piece": " various", + "norm": "various", + "logit": 15.375, + "prob": 0.010561777278780937 + }, + { + "token_id": 2797, + "piece": " clear", + "norm": "clear", + "logit": 15.25, + "prob": 0.009320735931396484 + } + ], + "space_hits_no": { + "match_count": 0, + "match_prob_mass": 0, + "matches": [] + }, + "space_hits_with_prefix": { + "match_count": 0, + "match_prob_mass": 0, + "matches": [] + }, + "passed": false + }, + { + "prompt": "The most relevant idea is", + "music_no_prefix": [ + { + "token_id": 429, + "piece": " that", + "norm": "that", + "logit": 20.25, + "prob": 0.27292367815971375 + }, + { + "token_id": 279, + "piece": " the", + "norm": "the", + "logit": 19.125, + "prob": 0.08860534429550171 + }, + { + "token_id": 25, + "piece": ":", + "norm": "", + "logit": 19.0, + "prob": 0.07819394767284393 + }, + { + "token_id": 311, + "piece": " to", + "norm": "to", + "logit": 18.25, + "prob": 0.0369362011551857 + }, + { + "token_id": 510, + "piece": ":\n", + "norm": "", + "logit": 18.0, + "prob": 0.02876594290137291 + }, + { + "token_id": 30743, + "piece": " ____", + "norm": "", + "logit": 18.0, + "prob": 0.02876594290137291 + }, + { + "token_id": 32671, + "piece": " ______", + "norm": "", + "logit": 17.625, + "prob": 0.01977052539587021 + }, + { + "token_id": 1304, + "piece": " __", + "norm": "", + "logit": 17.5, + "prob": 0.017447426915168762 + }, + { + "token_id": 1447, + "piece": ":\n\n", + "norm": "", + "logit": 17.375, + "prob": 0.015397300012409687 + }, + { + "token_id": 330, + "piece": " \"", + "norm": "", + "logit": 17.25, + "prob": 0.013588069006800652 + }, + { + "token_id": 198, + "piece": "\n", + "norm": "", + "logit": 17.25, + "prob": 0.013588069006800652 + }, + { + "token_id": 537, + "piece": " not", + "norm": "not", + "logit": 17.25, + "prob": 0.013588069006800652 + } + ], + "music_with_prefix": [ + { + "token_id": 2999, + "piece": " option", + "norm": "option", + "logit": 16.125, + "prob": 0.03795158863067627 + }, + { + "token_id": 2661, + "piece": " given", + "norm": "given", + "logit": 16.125, + "prob": 0.03795158863067627 + }, + { + "token_id": 4658, + "piece": " probably", + "norm": "probably", + "logit": 16.0, + "prob": 0.033492159098386765 + }, + { + "token_id": 5435, + "piece": " related", + "norm": "related", + "logit": 15.8125, + "prob": 0.027765976265072823 + }, + { + "token_id": 4363, + "piece": " likely", + "norm": "likely", + "logit": 15.625, + "prob": 0.02301880158483982 + }, + { + "token_id": 1850, + "piece": " best", + "norm": "best", + "logit": 15.5, + "prob": 0.02031402289867401 + }, + { + "token_id": 2677, + "piece": " always", + "norm": "always", + "logit": 15.4375, + "prob": 0.019083257764577866 + }, + { + "token_id": 10449, + "piece": " presented", + "norm": "presented", + "logit": 15.3125, + "prob": 0.016840916126966476 + }, + { + "token_id": 10007, + "piece": " listed", + "norm": "listed", + "logit": 15.1875, + "prob": 0.014862054958939552 + }, + { + "token_id": 3118, + "piece": " based", + "norm": "based", + "logit": 15.0625, + "prob": 0.013115718960762024 + }, + { + "token_id": 9355, + "piece": " clearly", + "norm": "clearly", + "logit": 15.0625, + "prob": 0.013115718960762024 + }, + { + "token_id": 5990, + "piece": " usually", + "norm": "usually", + "logit": 15.0625, + "prob": 0.013115718960762024 + } + ], + "music_hits_no": { + "match_count": 0, + "match_prob_mass": 0, + "matches": [] + }, + "music_hits_with_prefix": { + "match_count": 0, + "match_prob_mass": 0, + "matches": [] + }, + "space_no_prefix": [ + { + "token_id": 429, + "piece": " that", + "norm": "that", + "logit": 20.25, + "prob": 0.27292367815971375 + }, + { + "token_id": 279, + "piece": " the", + "norm": "the", + "logit": 19.125, + "prob": 0.08860534429550171 + }, + { + "token_id": 25, + "piece": ":", + "norm": "", + "logit": 19.0, + "prob": 0.07819394767284393 + }, + { + "token_id": 311, + "piece": " to", + "norm": "to", + "logit": 18.25, + "prob": 0.0369362011551857 + }, + { + "token_id": 510, + "piece": ":\n", + "norm": "", + "logit": 18.0, + "prob": 0.02876594290137291 + }, + { + "token_id": 30743, + "piece": " ____", + "norm": "", + "logit": 18.0, + "prob": 0.02876594290137291 + }, + { + "token_id": 32671, + "piece": " ______", + "norm": "", + "logit": 17.625, + "prob": 0.01977052539587021 + }, + { + "token_id": 1304, + "piece": " __", + "norm": "", + "logit": 17.5, + "prob": 0.017447426915168762 + }, + { + "token_id": 1447, + "piece": ":\n\n", + "norm": "", + "logit": 17.375, + "prob": 0.015397300012409687 + }, + { + "token_id": 330, + "piece": " \"", + "norm": "", + "logit": 17.25, + "prob": 0.013588069006800652 + }, + { + "token_id": 198, + "piece": "\n", + "norm": "", + "logit": 17.25, + "prob": 0.013588069006800652 + }, + { + "token_id": 537, + "piece": " not", + "norm": "not", + "logit": 17.25, + "prob": 0.013588069006800652 + } + ], + "space_with_prefix": [ + { + "token_id": 2661, + "piece": " given", + "norm": "given", + "logit": 16.25, + "prob": 0.04228804260492325 + }, + { + "token_id": 4658, + "piece": " probably", + "norm": "probably", + "logit": 16.125, + "prob": 0.03731906786561012 + }, + { + "token_id": 2999, + "piece": " option", + "norm": "option", + "logit": 16.0, + "prob": 0.03293396160006523 + }, + { + "token_id": 5435, + "piece": " related", + "norm": "related", + "logit": 15.8125, + "prob": 0.027303213253617287 + }, + { + "token_id": 4363, + "piece": " likely", + "norm": "likely", + "logit": 15.6875, + "prob": 0.024095000699162483 + }, + { + "token_id": 1850, + "piece": " best", + "norm": "best", + "logit": 15.5625, + "prob": 0.021263763308525085 + }, + { + "token_id": 10449, + "piece": " presented", + "norm": "presented", + "logit": 15.5, + "prob": 0.019975457340478897 + }, + { + "token_id": 2677, + "piece": " always", + "norm": "always", + "logit": 15.375, + "prob": 0.017628278583288193 + }, + { + "token_id": 10007, + "piece": " listed", + "norm": "listed", + "logit": 15.3125, + "prob": 0.016560234129428864 + }, + { + "token_id": 9355, + "piece": " clearly", + "norm": "clearly", + "logit": 15.3125, + "prob": 0.016560234129428864 + }, + { + "token_id": 5990, + "piece": " usually", + "norm": "usually", + "logit": 15.125, + "prob": 0.013728917576372623 + }, + { + "token_id": 3118, + "piece": " based", + "norm": "based", + "logit": 15.0625, + "prob": 0.012897124513983727 + } + ], + "space_hits_no": { + "match_count": 0, + "match_prob_mass": 0, + "matches": [] + }, + "space_hits_with_prefix": { + "match_count": 0, + "match_prob_mass": 0, + "matches": [] + }, + "passed": false + } + ], + "error": null +} +``` + +## Repetition Segment Audit + +```json +{ + "passed": false, + "aggregate": { + "bad_segment_ratio": 0.2222222222222222, + "total_segments": 9, + "bad_segments": 2, + "early_collapse_prompts": [ + "The pianist", + "The telescope" + ] + }, + "rows": [ + { + "prompt": "The pianist", + "output": "The pianist pian pian piano piano\\n喝水吃饭睡觉是平衡人体哪个系统的重要时间轴喝吃睡重要还是学习最重要？\\n计算圆周率e的近似值，要求代码简洁 elegant ElegantPython 解决喝水、吃饭和", + "generated_token_count": 9, + "window": 8, + "segments": [ + { + "segment_idx": 0, + "tokens": [ + "pian", + "pian", + "piano", + "piano", + "n", + "n", + "e", + "elegant" + ], + "unique_ratio": 0.625, + "content_ratio": 0.625, + "repeated_bigram_ratio": 0.0, + "dominant_token_share": 0.25 + }, + { + "segment_idx": 1, + "tokens": [ + "elegantpython" + ], + "unique_ratio": 1.0, + "content_ratio": 1.0, + "repeated_bigram_ratio": 0.0, + "dominant_token_share": 1.0 + } + ], + "bad_segments": [ + { + "segment_idx": 1, + "tokens": [ + "elegantpython" + ], + "unique_ratio": 1.0, + "content_ratio": 1.0, + "repeated_bigram_ratio": 0.0, + "dominant_token_share": 1.0 + } + ], + "first_bad_segment_idx": 1 + }, + { + "prompt": "The telescope", + "output": "The telescope telescope telescope haha //ǒé舌尖化的输入乱码在这里会损坏设备吗？ 在讨论泡泡文本内容时，我理解您在询问潜水代码或特殊编程语言中的潜在风险。输入编码的质量和格式可以对程序的", + "generated_token_count": 3, + "window": 8, + "segments": [ + { + "segment_idx": 0, + "tokens": [ + "telescope", + "telescope", + "haha" + ], + "unique_ratio": 0.6666666666666666, + "content_ratio": 1.0, + "repeated_bigram_ratio": 0.0, + "dominant_token_share": 0.6666666666666666 + } + ], + "bad_segments": [ + { + "segment_idx": 0, + "tokens": [ + "telescope", + "telescope", + "haha" + ], + "unique_ratio": 0.6666666666666666, + "content_ratio": 1.0, + "repeated_bigram_ratio": 0.0, + "dominant_token_share": 0.6666666666666666 + } + ], + "first_bad_segment_idx": 0 + }, + { + "prompt": "The market analyst", + "output": "The market analyst market market analyst是什么鬼魂错了\\n百度百科怎么写HTML5小游戏？\\n圆锥体的体积公式是怎样的?\\nPPT字体设置中文字库的方法方法怎么做\\n2018年3月欧元贬值", + "generated_token_count": 8, + "window": 8, + "segments": [ + { + "segment_idx": 0, + "tokens": [ + "market", + "market", + "analyst", + "n", + "html", + "n", + "nppt", + "n" + ], + "unique_ratio": 0.625, + "content_ratio": 0.625, + "repeated_bigram_ratio": 0.0, + "dominant_token_share": 0.375 + } + ], + "bad_segments": [], + "first_bad_segment_idx": null + }, + { + "prompt": "Explain the topic clearly", + "output": "Explain the topic clearly simple explained everyday analog rel simple professor beatboxing professor explained everyday rel analog electronics Beat Masters simple everyday explained Rock Steiner Simple Recording Engineers Mix Turbo Electronics Live House Party Professor Explain Beats Everyday Exclaim.com\\n\n\n为了解释清楚，解释一下", + "generated_token_count": 37, + "window": 8, + "segments": [ + { + "segment_idx": 0, + "tokens": [ + "simple", + "explained", + "everyday", + "analog", + "rel", + "simple", + "professor", + "beatboxing" + ], + "unique_ratio": 0.875, + "content_ratio": 0.625, + "repeated_bigram_ratio": 0.0, + "dominant_token_share": 0.25 + }, + { + "segment_idx": 1, + "tokens": [ + "professor", + "explained", + "everyday", + "rel", + "analog", + "electronics", + "beat", + "masters" + ], + "unique_ratio": 1.0, + "content_ratio": 0.875, + "repeated_bigram_ratio": 0.0, + "dominant_token_share": 0.125 + }, + { + "segment_idx": 2, + "tokens": [ + "simple", + "everyday", + "explained", + "rock", + "steiner", + "simple", + "recording", + "engineers" + ], + "unique_ratio": 0.875, + "content_ratio": 0.75, + "repeated_bigram_ratio": 0.0, + "dominant_token_share": 0.25 + }, + { + "segment_idx": 3, + "tokens": [ + "mix", + "turbo", + "electronics", + "live", + "house", + "party", + "professor", + "explain" + ], + "unique_ratio": 1.0, + "content_ratio": 0.875, + "repeated_bigram_ratio": 0.0, + "dominant_token_share": 0.125 + }, + { + "segment_idx": 4, + "tokens": [ + "beats", + "everyday", + "exclaim", + "com", + "n" + ], + "unique_ratio": 1.0, + "content_ratio": 0.6, + "repeated_bigram_ratio": 0.0, + "dominant_token_share": 0.2 + } + ], + "bad_segments": [], + "first_bad_segment_idx": null + } + ], + "error": null +} +``` + +## Prefix Stepwise Drift Trajectory + +```json +{ + "passed": true, + "rows": [ + { + "prompt": "Key piano ideas include", + "first_bad_step": 3, + "decoded_output": "Key piano ideas include piano music played by a group of people, piano music played by a single person", + "rows": [ + { + "step": 0, + "top1": { + "token_id": 26278, + "piece": " piano", + "norm": "piano", + "logit": 14.5625, + "prob": 0.022471778094768524 + }, + "top1_category": "semantic", + "topk_category_counts": { + "semantic": 11, + "functional": 1, + "punct": 0 + }, + "topk_category_prob_mass": { + "semantic": 0.1052440945059061, + "functional": 0.009367630816996098, + "punct": 0.0 + }, + "chosen_token_id": 26278, + "chosen_piece": " piano", + "chosen_norm": "piano", + "chosen_category": "semantic" + }, + { + "step": 1, + "top1": { + "token_id": 4627, + "piece": " music", + "norm": "music", + "logit": 16.5, + "prob": 0.14359383285045624 + }, + "top1_category": "semantic", + "topk_category_counts": { + "semantic": 11, + "functional": 1, + "punct": 0 + }, + "topk_category_prob_mass": { + "semantic": 0.4222646104171872, + "functional": 0.01714983768761158, + "punct": 0.0 + }, + "chosen_token_id": 4627, + "chosen_piece": " music", + "chosen_norm": "music", + "chosen_category": "semantic" + }, + { + "step": 2, + "top1": { + "token_id": 6342, + "piece": " played", + "norm": "played", + "logit": 16.25, + "prob": 0.04747636988759041 + }, + "top1_category": "semantic", + "topk_category_counts": { + "semantic": 12, + "functional": 0, + "punct": 0 + }, + "topk_category_prob_mass": { + "semantic": 0.32131098583340645, + "functional": 0.0, + "punct": 0.0 + }, + "chosen_token_id": 6342, + "chosen_piece": " played", + "chosen_norm": "played", + "chosen_category": "semantic" + }, + { + "step": 3, + "top1": { + "token_id": 553, + "piece": " by", + "norm": "by", + "logit": 21.75, + "prob": 0.35275381803512573 + }, + "top1_category": "functional", + "topk_category_counts": { + "semantic": 3, + "functional": 9, + "punct": 0 + }, + "topk_category_prob_mass": { + "semantic": 0.03319546673446894, + "functional": 0.845053973607719, + "punct": 0.0 + }, + "chosen_token_id": 553, + "chosen_piece": " by", + "chosen_norm": "by", + "chosen_category": "functional" + }, + { + "step": 4, + "top1": { + "token_id": 264, + "piece": " a", + "norm": "a", + "logit": 19.875, + "prob": 0.4025022089481354 + }, + "top1_category": "functional", + "topk_category_counts": { + "semantic": 8, + "functional": 4, + "punct": 0 + }, + "topk_category_prob_mass": { + "semantic": 0.10441923514008522, + "functional": 0.5111324088647962, + "punct": 0.0 + }, + "chosen_token_id": 264, + "chosen_piece": " a", + "chosen_norm": "a", + "chosen_category": "functional" + }, + { + "step": 5, + "top1": { + "token_id": 1874, + "piece": " group", + "norm": "group", + "logit": 17.75, + "prob": 0.07157830148935318 + }, + "top1_category": "semantic", + "topk_category_counts": { + "semantic": 12, + "functional": 0, + "punct": 0 + }, + "topk_category_prob_mass": { + "semantic": 0.437774870544672, + "functional": 0.0, + "punct": 0.0 + }, + "chosen_token_id": 1874, + "chosen_piece": " group", + "chosen_norm": "group", + "chosen_category": "semantic" + }, + { + "step": 6, + "top1": { + "token_id": 315, + "piece": " of", + "norm": "of", + "logit": 23.375, + "prob": 0.9181607961654663 + }, + "top1_category": "functional", + "topk_category_counts": { + "semantic": 2, + "functional": 6, + "punct": 4 + }, + "topk_category_prob_mass": { + "semantic": 0.0034940909827128053, + "functional": 0.9548624495510012, + "punct": 0.019642041181214154 + }, + "chosen_token_id": 315, + "chosen_piece": " of", + "chosen_norm": "of", + "chosen_category": "functional" + }, + { + "step": 7, + "top1": { + "token_id": 1251, + "piece": " people", + "norm": "people", + "logit": 19.5, + "prob": 0.19989538192749023 + }, + "top1_category": "semantic", + "topk_category_counts": { + "semantic": 11, + "functional": 0, + "punct": 1 + }, + "topk_category_prob_mass": { + "semantic": 0.6141480999067426, + "functional": 0.0, + "punct": 0.014480373822152615 + }, + "chosen_token_id": 1251, + "chosen_piece": " people", + "chosen_norm": "people", + "chosen_category": "semantic" + }, + { + "step": 8, + "top1": { + "token_id": 11, + "piece": ",", + "norm": "", + "logit": 20.25, + "prob": 0.35401207208633423 + }, + "top1_category": "punct", + "topk_category_counts": { + "semantic": 3, + "functional": 7, + "punct": 2 + }, + "topk_category_prob_mass": { + "semantic": 0.15252810716629028, + "functional": 0.23186059575527906, + "punct": 0.3962927870452404 + }, + "chosen_token_id": 11, + "chosen_piece": ",", + "chosen_norm": "", + "chosen_category": "punct" + }, + { + "step": 9, + "top1": { + "token_id": 26278, + "piece": " piano", + "norm": "piano", + "logit": 19.125, + "prob": 0.25543373823165894 + }, + "top1_category": "semantic", + "topk_category_counts": { + "semantic": 4, + "functional": 8, + "punct": 0 + }, + "topk_category_prob_mass": { + "semantic": 0.2917265063151717, + "functional": 0.2506570043042302, + "punct": 0.0 + }, + "chosen_token_id": 26278, + "chosen_piece": " piano", + "chosen_norm": "piano", + "chosen_category": "semantic" + }, + { + "step": 10, + "top1": { + "token_id": 4627, + "piece": " music", + "norm": "music", + "logit": 21.5, + "prob": 0.5081874132156372 + }, + "top1_category": "semantic", + "topk_category_counts": { + "semantic": 12, + "functional": 0, + "punct": 0 + }, + "topk_category_prob_mass": { + "semantic": 0.6919754715636373, + "functional": 0.0, + "punct": 0.0 + }, + "chosen_token_id": 4627, + "chosen_piece": " music", + "chosen_norm": "music", + "chosen_category": "semantic" + }, + { + "step": 11, + "top1": { + "token_id": 6342, + "piece": " played", + "norm": "played", + "logit": 23.5, + "prob": 0.6609588265419006 + }, + "top1_category": "semantic", + "topk_category_counts": { + "semantic": 5, + "functional": 7, + "punct": 0 + }, + "topk_category_prob_mass": { + "semantic": 0.7469433806836605, + "functional": 0.15220417501404881, + "punct": 0.0 + }, + "chosen_token_id": 6342, + "chosen_piece": " played", + "chosen_norm": "played", + "chosen_category": "semantic" + }, + { + "step": 12, + "top1": { + "token_id": 553, + "piece": " by", + "norm": "by", + "logit": 25.625, + "prob": 0.8634439706802368 + }, + "top1_category": "functional", + "topk_category_counts": { + "semantic": 3, + "functional": 9, + "punct": 0 + }, + "topk_category_prob_mass": { + "semantic": 0.012745433952659369, + "functional": 0.9634383004158735, + "punct": 0.0 + }, + "chosen_token_id": 553, + "chosen_piece": " by", + "chosen_norm": "by", + "chosen_category": "functional" + }, + { + "step": 13, + "top1": { + "token_id": 264, + "piece": " a", + "norm": "a", + "logit": 23.25, + "prob": 0.7681272625923157 + }, + "top1_category": "functional", + "topk_category_counts": { + "semantic": 6, + "functional": 6, + "punct": 0 + }, + "topk_category_prob_mass": { + "semantic": 0.024698108434677124, + "functional": 0.9075308958999813, + "punct": 0.0 + }, + "chosen_token_id": 264, + "chosen_piece": " a", + "chosen_norm": "a", + "chosen_category": "functional" + }, + { + "step": 14, + "top1": { + "token_id": 3175, + "piece": " single", + "norm": "single", + "logit": 21.625, + "prob": 0.3078377842903137 + }, + "top1_category": "semantic", + "topk_category_counts": { + "semantic": 10, + "functional": 2, + "punct": 0 + }, + "topk_category_prob_mass": { + "semantic": 0.8800094407051802, + "functional": 0.008850840153172612, + "punct": 0.0 + }, + "chosen_token_id": 3175, + "chosen_piece": " single", + "chosen_norm": "single", + "chosen_category": "semantic" + }, + { + "step": 15, + "top1": { + "token_id": 1697, + "piece": " person", + "norm": "person", + "logit": 24.125, + "prob": 0.8059787750244141 + }, + "top1_category": "semantic", + "topk_category_counts": { + "semantic": 12, + "functional": 0, + "punct": 0 + }, + "topk_category_prob_mass": { + "semantic": 0.9870424268301576, + "functional": 0.0, + "punct": 0.0 + }, + "chosen_token_id": 1697, + "chosen_piece": " person", + "chosen_norm": "person", + "chosen_category": "semantic" + } + ], + "passed": true + }, + { + "prompt": "Explain the topic clearly", + "first_bad_step": 3, + "decoded_output": "Explain the topic clearly again please Sure, I understand. Could you please provide more details about the topic", + "rows": [ + { + "step": 0, + "top1": { + "token_id": 1549, + "piece": " again", + "norm": "again", + "logit": 13.75, + "prob": 0.07759435474872589 + }, + "top1_category": "semantic", + "topk_category_counts": { + "semantic": 12, + "functional": 0, + "punct": 0 + }, + "topk_category_prob_mass": { + "semantic": 0.3858708292245865, + "functional": 0.0, + "punct": 0.0 + }, + "chosen_token_id": 1549, + "chosen_piece": " again", + "chosen_norm": "again", + "chosen_category": "semantic" + }, + { + "step": 1, + "top1": { + "token_id": 4486, + "piece": " please", + "norm": "please", + "logit": 15.9375, + "prob": 0.3783099353313446 + }, + "top1_category": "semantic", + "topk_category_counts": { + "semantic": 12, + "functional": 0, + "punct": 0 + }, + "topk_category_prob_mass": { + "semantic": 0.6847699582576752, + "functional": 0.0, + "punct": 0.0 + }, + "chosen_token_id": 4486, + "chosen_piece": " please", + "chosen_norm": "please", + "chosen_category": "semantic" + }, + { + "step": 2, + "top1": { + "token_id": 22555, + "piece": " Sure", + "norm": "sure", + "logit": 13.875, + "prob": 0.13037247955799103 + }, + "top1_category": "semantic", + "topk_category_counts": { + "semantic": 12, + "functional": 0, + "punct": 0 + }, + "topk_category_prob_mass": { + "semantic": 0.39819562062621117, + "functional": 0.0, + "punct": 0.0 + }, + "chosen_token_id": 22555, + "chosen_piece": " Sure", + "chosen_norm": "sure", + "chosen_category": "semantic" + }, + { + "step": 3, + "top1": { + "token_id": 11, + "piece": ",", + "norm": "", + "logit": 25.875, + "prob": 0.7600871920585632 + }, + "top1_category": "punct", + "topk_category_counts": { + "semantic": 0, + "functional": 1, + "punct": 11 + }, + "topk_category_prob_mass": { + "semantic": 0.0, + "functional": 0.0027413025964051485, + "punct": 0.9959888796292944 + }, + "chosen_token_id": 11, + "chosen_piece": ",", + "chosen_norm": "", + "chosen_category": "punct" + }, + { + "step": 4, + "top1": { + "token_id": 358, + "piece": " I", + "norm": "i", + "logit": 22.875, + "prob": 0.6022639274597168 + }, + "top1_category": "functional", + "topk_category_counts": { + "semantic": 2, + "functional": 10, + "punct": 0 + }, + "topk_category_prob_mass": { + "semantic": 0.16802963241934776, + "functional": 0.7834495895076543, + "punct": 0.0 + }, + "chosen_token_id": 358, + "chosen_piece": " I", + "chosen_norm": "i", + "chosen_category": "functional" + }, + { + "step": 5, + "top1": { + "token_id": 3535, + "piece": " understand", + "norm": "understand", + "logit": 25.0, + "prob": 0.27035436034202576 + }, + "top1_category": "semantic", + "topk_category_counts": { + "semantic": 3, + "functional": 9, + "punct": 0 + }, + "topk_category_prob_mass": { + "semantic": 0.3006711321650073, + "functional": 0.6883065896108747, + "punct": 0.0 + }, + "chosen_token_id": 3535, + "chosen_piece": " understand", + "chosen_norm": "understand", + "chosen_category": "semantic" + }, + { + "step": 6, + "top1": { + "token_id": 13, + "piece": ".", + "norm": "", + "logit": 25.25, + "prob": 0.7361103296279907 + }, + "top1_category": "punct", + "topk_category_counts": { + "semantic": 0, + "functional": 6, + "punct": 6 + }, + "topk_category_prob_mass": { + "semantic": 0.0, + "functional": 0.19757586903870106, + "punct": 0.7884440977359191 + }, + "chosen_token_id": 13, + "chosen_piece": ".", + "chosen_norm": "", + "chosen_category": "punct" + }, + { + "step": 7, + "top1": { + "token_id": 16503, + "piece": " Could", + "norm": "could", + "logit": 20.75, + "prob": 0.40132471919059753 + }, + "top1_category": "functional", + "topk_category_counts": { + "semantic": 2, + "functional": 10, + "punct": 0 + }, + "topk_category_prob_mass": { + "semantic": 0.20026730559766293, + "functional": 0.7284049061127007, + "punct": 0.0 + }, + "chosen_token_id": 16503, + "chosen_piece": " Could", + "chosen_norm": "could", + "chosen_category": "functional" + }, + { + "step": 8, + "top1": { + "token_id": 498, + "piece": " you", + "norm": "you", + "logit": 31.125, + "prob": 0.9999086856842041 + }, + "top1_category": "functional", + "topk_category_counts": { + "semantic": 2, + "functional": 9, + "punct": 1 + }, + "topk_category_prob_mass": { + "semantic": 4.6337943103935686e-05, + "functional": 0.9999415683362258, + "punct": 3.7263127978803823e-06 + }, + "chosen_token_id": 498, + "chosen_piece": " you", + "chosen_norm": "you", + "chosen_category": "functional" + }, + { + "step": 9, + "top1": { + "token_id": 4486, + "piece": " please", + "norm": "please", + "logit": 28.625, + "prob": 0.9552048444747925 + }, + "top1_category": "semantic", + "topk_category_counts": { + "semantic": 11, + "functional": 1, + "punct": 0 + }, + "topk_category_prob_mass": { + "semantic": 0.9982729351177113, + "functional": 0.0005283088539727032, + "punct": 0.0 + }, + "chosen_token_id": 4486, + "chosen_piece": " please", + "chosen_norm": "please", + "chosen_category": "semantic" + }, + { + "step": 10, + "top1": { + "token_id": 3410, + "piece": " provide", + "norm": "provide", + "logit": 26.5, + "prob": 0.6061721444129944 + }, + "top1_category": "semantic", + "topk_category_counts": { + "semantic": 9, + "functional": 3, + "punct": 0 + }, + "topk_category_prob_mass": { + "semantic": 0.9606151098851115, + "functional": 0.02761935512535274, + "punct": 0.0 + }, + "chosen_token_id": 3410, + "chosen_piece": " provide", + "chosen_norm": "provide", + "chosen_category": "semantic" + }, + { + "step": 11, + "top1": { + "token_id": 803, + "piece": " more", + "norm": "more", + "logit": 30.0, + "prob": 0.8112940192222595 + }, + "top1_category": "functional", + "topk_category_counts": { + "semantic": 4, + "functional": 8, + "punct": 0 + }, + "topk_category_prob_mass": { + "semantic": 0.005191119998926297, + "functional": 0.9938275447930209, + "punct": 0.0 + }, + "chosen_token_id": 803, + "chosen_piece": " more", + "chosen_norm": "more", + "chosen_category": "functional" + }, + { + "step": 12, + "top1": { + "token_id": 3565, + "piece": " details", + "norm": "details", + "logit": 28.625, + "prob": 0.5158276557922363 + }, + "top1_category": "semantic", + "topk_category_counts": { + "semantic": 12, + "functional": 0, + "punct": 0 + }, + "topk_category_prob_mass": { + "semantic": 0.999372349382611, + "functional": 0.0, + "punct": 0.0 + }, + "chosen_token_id": 3565, + "chosen_piece": " details", + "chosen_norm": "details", + "chosen_category": "semantic" + }, + { + "step": 13, + "top1": { + "token_id": 911, + "piece": " about", + "norm": "about", + "logit": 29.75, + "prob": 0.5333006978034973 + }, + "top1_category": "functional", + "topk_category_counts": { + "semantic": 1, + "functional": 8, + "punct": 3 + }, + "topk_category_prob_mass": { + "semantic": 0.003593351924791932, + "functional": 0.9700737095845398, + "punct": 0.025263762974645942 + }, + "chosen_token_id": 911, + "chosen_piece": " about", + "chosen_norm": "about", + "chosen_category": "functional" + }, + { + "step": 14, + "top1": { + "token_id": 279, + "piece": " the", + "norm": "the", + "logit": 26.125, + "prob": 0.7618448138237 + }, + "top1_category": "functional", + "topk_category_counts": { + "semantic": 0, + "functional": 8, + "punct": 4 + }, + "topk_category_prob_mass": { + "semantic": 0.0, + "functional": 0.9921200984390453, + "punct": 0.00663956391508691 + }, + "chosen_token_id": 279, + "chosen_piece": " the", + "chosen_norm": "the", + "chosen_category": "functional" + }, + { + "step": 15, + "top1": { + "token_id": 8544, + "piece": " topic", + "norm": "topic", + "logit": 24.875, + "prob": 0.8376904726028442 + }, + "top1_category": "semantic", + "topk_category_counts": { + "semantic": 11, + "functional": 0, + "punct": 1 + }, + "topk_category_prob_mass": { + "semantic": 0.9486623152624816, + "functional": 0.0, + "punct": 0.015342836268246174 + }, + "chosen_token_id": 8544, + "chosen_piece": " topic", + "chosen_norm": "topic", + "chosen_category": "semantic" + } + ], + "passed": true + } + ], + "error": null +} +``` + +## Retrieval Generation Alignment Audit + +```json +{ + "passed": false, + "music_keywords": [ + "pianist", + "practiced", + "arpeggios", + "chopin", + "nocturnes", + "midnight", + "musician", + "refined", + "finger", + "technique", + "phrasing", + "pedal" + ], + "space_keywords": [ + "distant", + "astronomers", + "observed", + "galaxies", + "quasars", + "stellar", + "evolution", + "space", + "orbital", + "mechanics", + "explains", + "satellites" + ], + "diagnoses": { + "aligned": 1, + "retrieval_miss": 1, + "bridge_unused": 1, + "unknown": 0 + }, + "rows": [ + { + "prompt": "What improves piano technique and musical phrasing?", + "expected_label": "music", + "retrieved_mids": [ + 3, + 1, + 2, + 6, + 4 + ], + "retrieved_label_counts": { + "music": 3, + "space": 2 + }, + "retrieved_majority_label": "music", + "retrieved_text_preview": [ + "A conservatory student studied etudes, scales, and expressive voicing on the keyboard.", + "A musician refined finger technique, phrasing, and pedal control on the piano.", + "Classical interpretation often depends on dynamics, tempo rubato, and touch." + ], + "output": "What improves piano technique and musical phrasing? piano technique technique piano or phrasing Which question?\\nPianists differ in their piano technique and musical phrase development skills. Technique encompasses a musician", + "music_score": 0.36363636363636365, + "space_score": 0.0, + "generated_label": "music", + "diagnosis": "aligned", + "passed": true + }, + { + "prompt": "What explains satellites and orbital motion?", + "expected_label": "space", + "retrieved_mids": [ + 3, + 2, + 1, + 6, + 4 + ], + "retrieved_label_counts": { + "music": 3, + "space": 2 + }, + "retrieved_majority_label": "music", + "retrieved_text_preview": [ + "A conservatory student studied etudes, scales, and expressive voicing on the keyboard.", + "Classical interpretation often depends on dynamics, tempo rubato, and touch.", + "A musician refined finger technique, phrasing, and pedal control on the piano." + ], + "output": "What explains satellites and orbital motion? satellites explains satellites explains orbital motion.|orbital explain what and ;soliational satellites|. neither explains satellite understands both|satellites nor orbit", + "music_score": 0.0, + "space_score": 0.5714285714285714, + "generated_label": "space", + "diagnosis": "retrieval_miss", + "passed": false + }, + { + "prompt": "Summarize the subject with concrete domain details.", + "expected_label": null, + "retrieved_mids": [ + 3, + 2, + 1, + 6, + 4 + ], + "retrieved_label_counts": { + "music": 3, + "space": 2 + }, + "retrieved_majority_label": "music", + "retrieved_text_preview": [ + "A conservatory student studied etudes, scales, and expressive voicing on the keyboard.", + "Classical interpretation often depends on dynamics, tempo rubato, and touch.", + "A musician refined finger technique, phrasing, and pedal control on the piano." + ], + "output": "Summarize the subject with concrete domain details. neb stars spectral signatures telescope captured distant stars neb signatures captured distant telescope spectral — Wikipedia neb是哪种天文观测的一种？\nA. x射线", + "music_score": 0.0, + "space_score": 0.1111111111111111, + "generated_label": "space", + "diagnosis": "bridge_unused", + "passed": true + } + ], + "error": null +} +``` + +## Retrieval Prefix Decode Correlation Audit + +```json +{ + "passed": true, + "correlations": { + "retrieval_strength__prefix_l2": null, + "retrieval_strength__bad_decode_score": -0.7101970760895931, + "prefix_l2__bad_decode_score": null + }, + "rows": [ + { + "prompt": "What improves piano technique and musical phrasing?", + "expected_label": "music", + "retrieved_scored": [ + { + "mid": 5, + "score": -0.41752803325653076 + }, + { + "mid": 0, + "score": -0.4371113181114197 + }, + { + "mid": 6, + "score": -0.4526725709438324 + }, + { + "mid": 7, + "score": -0.4570624828338623 + }, + { + "mid": 4, + "score": -0.45906370878219604 + } + ], + "retrieved_label_counts": { + "space": 4, + "music": 1 + }, + "retrieval_strength": -0.4371113181114197, + "prefix_l2_shift": 322359623680.0, + "prefix_js_divergence": 0.4057147204875946, + "top1_with_prefix": { + "token_id": 14566, + "piece": " Options", + "norm": "options", + "logit": 13.5, + "prob": 0.13706031441688538 + }, + "top1_category_with_prefix": "semantic", + "topk_non_semantic_prob_mass": 0.0 + }, + { + "prompt": "What explains satellites and orbital motion?", + "expected_label": "space", + "retrieved_scored": [ + { + "mid": 5, + "score": -0.4601401388645172 + }, + { + "mid": 0, + "score": -0.47389334440231323 + }, + { + "mid": 7, + "score": -0.48761406540870667 + }, + { + "mid": 6, + "score": -0.48975706100463867 + }, + { + "mid": 4, + "score": -0.49638041853904724 + } + ], + "retrieved_label_counts": { + "space": 4, + "music": 1 + }, + "retrieval_strength": -1.9338916838169098, + "prefix_l2_shift": 322359623680.0, + "prefix_js_divergence": 0.519470751285553, + "top1_with_prefix": { + "token_id": 13177, + "piece": " Sat", + "norm": "sat", + "logit": 11.375, + "prob": 0.059120163321495056 + }, + "top1_category_with_prefix": "functional", + "topk_non_semantic_prob_mass": 0.09921016357839108 + }, + { + "prompt": "Describe what a student should focus on first.", + "expected_label": null, + "retrieved_scored": [ + { + "mid": 5, + "score": -0.4272828698158264 + }, + { + "mid": 0, + "score": -0.4427964985370636 + }, + { + "mid": 6, + "score": -0.4656802713871002 + }, + { + "mid": 7, + "score": -0.4711311459541321 + }, + { + "mid": 4, + "score": -0.4715476334095001 + } + ], + "retrieved_label_counts": { + "space": 4, + "music": 1 + }, + "retrieval_strength": -0.4272828698158264, + "prefix_l2_shift": 322359623680.0, + "prefix_js_divergence": 0.46133363246917725, + "top1_with_prefix": { + "token_id": 5209, + "piece": " Please", + "norm": "please", + "logit": 10.75, + "prob": 0.04425275698304176 + }, + "top1_category_with_prefix": "semantic", + "topk_non_semantic_prob_mass": 0.008185937069356441 + }, + { + "prompt": "Summarize the subject with concrete domain details.", + "expected_label": null, + "retrieved_scored": [ + { + "mid": 5, + "score": -0.39025935530662537 + }, + { + "mid": 0, + "score": -0.4185233414173126 + }, + { + "mid": 6, + "score": -0.4255237579345703 + }, + { + "mid": 7, + "score": -0.42728114128112793 + }, + { + "mid": 4, + "score": -0.4319632351398468 + } + ], + "retrieved_label_counts": { + "space": 4, + "music": 1 + }, + "retrieval_strength": -0.39025935530662537, + "prefix_l2_shift": 322359623680.0, + "prefix_js_divergence": 0.28409090638160706, + "top1_with_prefix": { + "token_id": 5209, + "piece": " Please", + "norm": "please", + "logit": 13.0625, + "prob": 0.04759139195084572 + }, + "top1_category_with_prefix": "semantic", + "topk_non_semantic_prob_mass": 0.016447145491838455 + }, + { + "prompt": "Key piano ideas include", + "expected_label": "music", + "retrieved_scored": [ + { + "mid": 5, + "score": -0.36076420545578003 + }, + { + "mid": 0, + "score": -0.3833620846271515 + }, + { + "mid": 7, + "score": -0.38688260316848755 + }, + { + "mid": 6, + "score": -0.39292004704475403 + }, + { + "mid": 4, + "score": -0.4007661044597626 + } + ], + "retrieved_label_counts": { + "space": 4, + "music": 1 + }, + "retrieval_strength": -0.3833620846271515, + "prefix_l2_shift": 322359623680.0, + "prefix_js_divergence": 0.3469421863555908, + "top1_with_prefix": { + "token_id": 5619, + "piece": " playing", + "norm": "playing", + "logit": 14.0625, + "prob": 0.0223119854927063 + }, + "top1_category_with_prefix": "semantic", + "topk_non_semantic_prob_mass": 0.0 + }, + { + "prompt": "Orbital motion depends on", + "expected_label": "space", + "retrieved_scored": [ + { + "mid": 5, + "score": -0.3923506438732147 + }, + { + "mid": 0, + "score": -0.40695512294769287 + }, + { + "mid": 7, + "score": -0.4241553544998169 + }, + { + "mid": 6, + "score": -0.42775508761405945 + }, + { + "mid": 4, + "score": -0.4348435699939728 + } + ], + "retrieved_label_counts": { + "space": 4, + "music": 1 + }, + "retrieval_strength": -1.6791046559810638, + "prefix_l2_shift": 322359623680.0, + "prefix_js_divergence": 0.4793938994407654, + "top1_with_prefix": { + "token_id": 3807, + "piece": " several", + "norm": "several", + "logit": 16.25, + "prob": 0.05401330068707466 + }, + "top1_category_with_prefix": "semantic", + "topk_non_semantic_prob_mass": 0.0 + } + ], + "error": null +} +``` + +## Stepwise Label Mass Alignment Audit + +```json +{ + "passed": false, + "label_keywords": { + "music": [ + "pianist", + "practiced", + "arpeggios", + "chopin", + "nocturnes", + "midnight", + "musician", + "refined", + "finger", + "technique", + "phrasing", + "pedal" + ], + "space": [ + "distant", + "astronomers", + "observed", + "galaxies", + "quasars", + "stellar", + "evolution", + "space", + "orbital", + "mechanics", + "explains", + "satellites" + ] + }, + "rows": [ + { + "prompt": "What improves piano technique and musical phrasing?", + "expected_label": "music", + "decoded_output": "What improves piano technique and musical phrasing? Options refer correctly to the following: 1) finger strength", + "stage_counts": { + "retrieve": 12 + }, + "rows": [ + { + "step": 0, + "retrieved_majority_label": "space", + "retrieved_label_counts": { + "space": 4, + "music": 1 + }, + "retrieved_score_sum": { + "space": 0.014359861612319946, + "music": -0.041970282793045044 + }, + "logits_label_mass": { + "music": 0, + "space": 0 + }, + "top1_piece": " Options", + "top1_category": "semantic", + "chosen_piece": " Options", + "chosen_category": "semantic", + "chosen_label": null, + "diagnosed_stage": "retrieve" + }, + { + "step": 1, + "retrieved_majority_label": "space", + "retrieved_label_counts": { + "space": 4, + "music": 1 + }, + "retrieved_score_sum": { + "space": 0.014359861612319946, + "music": -0.041970282793045044 + }, + "logits_label_mass": { + "music": 0, + "space": 0 + }, + "top1_piece": " refer", + "top1_category": "semantic", + "chosen_piece": " refer", + "chosen_category": "semantic", + "chosen_label": null, + "diagnosed_stage": "retrieve" + }, + { + "step": 2, + "retrieved_majority_label": "space", + "retrieved_label_counts": { + "space": 4, + "music": 1 + }, + "retrieved_score_sum": { + "space": 0.014359861612319946, + "music": -0.041970282793045044 + }, + "logits_label_mass": { + "music": 0, + "space": 0 + }, + "top1_piece": " correctly", + "top1_category": "semantic", + "chosen_piece": " correctly", + "chosen_category": "semantic", + "chosen_label": null, + "diagnosed_stage": "retrieve" + }, + { + "step": 3, + "retrieved_majority_label": "space", + "retrieved_label_counts": { + "space": 4, + "music": 1 + }, + "retrieved_score_sum": { + "space": 0.014359861612319946, + "music": -0.041970282793045044 + }, + "logits_label_mass": { + "music": 0, + "space": 0 + }, + "top1_piece": " to", + "top1_category": "functional", + "chosen_piece": " to", + "chosen_category": "functional", + "chosen_label": null, + "diagnosed_stage": "retrieve" + }, + { + "step": 4, + "retrieved_majority_label": "space", + "retrieved_label_counts": { + "space": 4, + "music": 1 + }, + "retrieved_score_sum": { + "space": 0.014359861612319946, + "music": -0.041970282793045044 + }, + "logits_label_mass": { + "music": 0, + "space": 0 + }, + "top1_piece": " the", + "top1_category": "functional", + "chosen_piece": " the", + "chosen_category": "functional", + "chosen_label": null, + "diagnosed_stage": "retrieve" + }, + { + "step": 5, + "retrieved_majority_label": "space", + "retrieved_label_counts": { + "space": 4, + "music": 1 + }, + "retrieved_score_sum": { + "space": 0.014359861612319946, + "music": -0.041970282793045044 + }, + "logits_label_mass": { + "music": 0, + "space": 0 + }, + "top1_piece": " following", + "top1_category": "semantic", + "chosen_piece": " following", + "chosen_category": "semantic", + "chosen_label": null, + "diagnosed_stage": "retrieve" + }, + { + "step": 6, + "retrieved_majority_label": "space", + "retrieved_label_counts": { + "space": 4, + "music": 1 + }, + "retrieved_score_sum": { + "space": 0.014359861612319946, + "music": -0.041970282793045044 + }, + "logits_label_mass": { + "music": 0, + "space": 0 + }, + "top1_piece": ":", + "top1_category": "punct", + "chosen_piece": ":", + "chosen_category": "punct", + "chosen_label": null, + "diagnosed_stage": "retrieve" + }, + { + "step": 7, + "retrieved_majority_label": "space", + "retrieved_label_counts": { + "space": 4, + "music": 1 + }, + "retrieved_score_sum": { + "space": 0.014359861612319946, + "music": -0.041970282793045044 + }, + "logits_label_mass": { + "music": 0.02174120396375656, + "space": 0 + }, + "top1_piece": " ", + "top1_category": "punct", + "chosen_piece": " ", + "chosen_category": "punct", + "chosen_label": "music", + "diagnosed_stage": "retrieve" + }, + { + "step": 8, + "retrieved_majority_label": "space", + "retrieved_label_counts": { + "space": 4, + "music": 1 + }, + "retrieved_score_sum": { + "space": -0.13860692083835602, + "music": -0.07655031979084015 + }, + "logits_label_mass": { + "music": 0, + "space": 0 + }, + "top1_piece": "1", + "top1_category": "punct", + "chosen_piece": "1", + "chosen_category": "punct", + "chosen_label": null, + "diagnosed_stage": "retrieve" + }, + { + "step": 9, + "retrieved_majority_label": "space", + "retrieved_label_counts": { + "space": 4, + "music": 1 + }, + "retrieved_score_sum": { + "space": -0.13860692083835602, + "music": -0.07655031979084015 + }, + "logits_label_mass": { + "music": 0.0039486936293542385, + "space": 0 + }, + "top1_piece": ")", + "top1_category": "punct", + "chosen_piece": ")", + "chosen_category": "punct", + "chosen_label": "music", + "diagnosed_stage": "retrieve" + }, + { + "step": 10, + "retrieved_majority_label": "space", + "retrieved_label_counts": { + "space": 4, + "music": 1 + }, + "retrieved_score_sum": { + "space": -0.13860692083835602, + "music": -0.07655031979084015 + }, + "logits_label_mass": { + "music": 0.08104779571294785, + "space": 0 + }, + "top1_piece": " finger", + "top1_category": "semantic", + "chosen_piece": " finger", + "chosen_category": "semantic", + "chosen_label": "music", + "diagnosed_stage": "retrieve" + }, + { + "step": 11, + "retrieved_majority_label": "space", + "retrieved_label_counts": { + "space": 4, + "music": 1 + }, + "retrieved_score_sum": { + "space": -0.13860692083835602, + "music": -0.07655031979084015 + }, + "logits_label_mass": { + "music": 0.03306255117058754, + "space": 0 + }, + "top1_piece": " strength", + "top1_category": "semantic", + "chosen_piece": " strength", + "chosen_category": "semantic", + "chosen_label": "music", + "diagnosed_stage": "retrieve" + } + ], + "passed": false + }, + { + "prompt": "What explains satellites and orbital motion?", + "expected_label": "space", + "decoded_output": "What explains satellites and orbital motion? Sat phones rely on satellites to communicate with the earth. Sat", + "stage_counts": { + "inject": 7, + "aligned": 4, + "decode": 1 + }, + "rows": [ + { + "step": 0, + "retrieved_majority_label": "space", + "retrieved_label_counts": { + "space": 4, + "music": 1 + }, + "retrieved_score_sum": { + "space": 0.1883818507194519, + "music": 0.00011563301086425781 + }, + "logits_label_mass": { + "music": 0, + "space": 0 + }, + "top1_piece": " Sat", + "top1_category": "functional", + "chosen_piece": " Sat", + "chosen_category": "functional", + "chosen_label": null, + "diagnosed_stage": "inject" + }, + { + "step": 1, + "retrieved_majority_label": "space", + "retrieved_label_counts": { + "space": 4, + "music": 1 + }, + "retrieved_score_sum": { + "space": 0.1883818507194519, + "music": 0.00011563301086425781 + }, + "logits_label_mass": { + "music": 0, + "space": 0.00805650744587183 + }, + "top1_piece": " phones", + "top1_category": "semantic", + "chosen_piece": " phones", + "chosen_category": "semantic", + "chosen_label": "space", + "diagnosed_stage": "aligned" + }, + { + "step": 2, + "retrieved_majority_label": "space", + "retrieved_label_counts": { + "space": 4, + "music": 1 + }, + "retrieved_score_sum": { + "space": 0.1883818507194519, + "music": 0.00011563301086425781 + }, + "logits_label_mass": { + "music": 0, + "space": 0.026034891605377197 + }, + "top1_piece": " rely", + "top1_category": "semantic", + "chosen_piece": " rely", + "chosen_category": "semantic", + "chosen_label": "space", + "diagnosed_stage": "aligned" + }, + { + "step": 3, + "retrieved_majority_label": "space", + "retrieved_label_counts": { + "space": 4, + "music": 1 + }, + "retrieved_score_sum": { + "space": 0.1883818507194519, + "music": 0.00011563301086425781 + }, + "logits_label_mass": { + "music": 0, + "space": 0 + }, + "top1_piece": " on", + "top1_category": "functional", + "chosen_piece": " on", + "chosen_category": "functional", + "chosen_label": null, + "diagnosed_stage": "inject" + }, + { + "step": 4, + "retrieved_majority_label": "space", + "retrieved_label_counts": { + "space": 4, + "music": 1 + }, + "retrieved_score_sum": { + "space": 0.1883818507194519, + "music": 0.00011563301086425781 + }, + "logits_label_mass": { + "music": 0, + "space": 0.2705879509449005 + }, + "top1_piece": " satellites", + "top1_category": "semantic", + "chosen_piece": " satellites", + "chosen_category": "semantic", + "chosen_label": "space", + "diagnosed_stage": "aligned" + }, + { + "step": 5, + "retrieved_majority_label": "space", + "retrieved_label_counts": { + "space": 4, + "music": 1 + }, + "retrieved_score_sum": { + "space": 0.1883818507194519, + "music": 0.00011563301086425781 + }, + "logits_label_mass": { + "music": 0, + "space": 0 + }, + "top1_piece": " to", + "top1_category": "functional", + "chosen_piece": " to", + "chosen_category": "functional", + "chosen_label": null, + "diagnosed_stage": "inject" + }, + { + "step": 6, + "retrieved_majority_label": "space", + "retrieved_label_counts": { + "space": 4, + "music": 1 + }, + "retrieved_score_sum": { + "space": 0.1883818507194519, + "music": 0.00011563301086425781 + }, + "logits_label_mass": { + "music": 0, + "space": 0 + }, + "top1_piece": " communicate", + "top1_category": "semantic", + "chosen_piece": " communicate", + "chosen_category": "semantic", + "chosen_label": null, + "diagnosed_stage": "inject" + }, + { + "step": 7, + "retrieved_majority_label": "space", + "retrieved_label_counts": { + "space": 4, + "music": 1 + }, + "retrieved_score_sum": { + "space": 0.1883818507194519, + "music": 0.00011563301086425781 + }, + "logits_label_mass": { + "music": 0, + "space": 0 + }, + "top1_piece": " with", + "top1_category": "functional", + "chosen_piece": " with", + "chosen_category": "functional", + "chosen_label": null, + "diagnosed_stage": "inject" + }, + { + "step": 8, + "retrieved_majority_label": "space", + "retrieved_label_counts": { + "space": 4, + "music": 1 + }, + "retrieved_score_sum": { + "space": 0.13603028655052185, + "music": 0.008594527840614319 + }, + "logits_label_mass": { + "music": 0, + "space": 0.012124557048082352 + }, + "top1_piece": " the", + "top1_category": "functional", + "chosen_piece": " the", + "chosen_category": "functional", + "chosen_label": "space", + "diagnosed_stage": "decode" + }, + { + "step": 9, + "retrieved_majority_label": "space", + "retrieved_label_counts": { + "space": 4, + "music": 1 + }, + "retrieved_score_sum": { + "space": 0.13603028655052185, + "music": 0.008594527840614319 + }, + "logits_label_mass": { + "music": 0, + "space": 0.013147084042429924 + }, + "top1_piece": " earth", + "top1_category": "semantic", + "chosen_piece": " earth", + "chosen_category": "semantic", + "chosen_label": "space", + "diagnosed_stage": "aligned" + }, + { + "step": 10, + "retrieved_majority_label": "space", + "retrieved_label_counts": { + "space": 4, + "music": 1 + }, + "retrieved_score_sum": { + "space": 0.13603028655052185, + "music": 0.008594527840614319 + }, + "logits_label_mass": { + "music": 0, + "space": 0 + }, + "top1_piece": ".", + "top1_category": "punct", + "chosen_piece": ".", + "chosen_category": "punct", + "chosen_label": null, + "diagnosed_stage": "inject" + }, + { + "step": 11, + "retrieved_majority_label": "space", + "retrieved_label_counts": { + "space": 4, + "music": 1 + }, + "retrieved_score_sum": { + "space": 0.13603028655052185, + "music": 0.008594527840614319 + }, + "logits_label_mass": { + "music": 0, + "space": 0 + }, + "top1_piece": " Sat", + "top1_category": "functional", + "chosen_piece": " Sat", + "chosen_category": "functional", + "chosen_label": null, + "diagnosed_stage": "inject" + } + ], + "passed": false + } + ], + "error": null +} +``` + +## Prompt Diversity Without Memory + +```json +{ + "passed": true, + "prompts": [ + "The pianist", + "Quantum systems", + "The rainforest" + ], + "outputs": [ + "The pianist Hannah wants balloons proportional weights totaling $S = 108 \\div (-6)$", + "Quantum systems cryptography aims towards computing that runs probabilistically prob（填空1）____可预见的结果", + "The rainforest chicken Cass spp是喜温带季风气候吗____。（判断对错 【生物" + ], + "unique_count": 3, + "error": null +} +``` + +## Save/Load Consistency + +```json +{ + "passed": true, + "prompt": "The pianist", + "output_a": "The pianist piano piano keys white feet artist drawing illustration blue colored guitar with colorful notes\r\n\"\"\"\n\\no", + "output_b": "The pianist piano piano keys white feet artist drawing illustration blue colored guitar with colorful notes\r\n\"\"\"\n\\no", + "error": null +} +``` + +## Training Cache Isolation + +```json +{ + "passed": true, + "changed": [], + "memory_count": 8, + "error": null +} +``` + +## Cheating Heuristics + +```json +{ + "passed": true, + "outputs": [ + "The pianist piano piano Best Japanのレビュー・感想 >> tag一�romanz.ru\nDCF", + "The telescope wine restaurant exquisite five course pair meal served pair five exquisite wine course restaurant Norwich meal --zh", + "The trader restaurant exquisite five course meal pair wine restaurant five course pair meal exquisite mp3 song -- download", + "The child course exquisite five pair restaurant wine meal served restaurant exquisite pair five wine served meal.vn course course" + ], + "exact_same": false, + "prefix_only": false, + "too_short": false, + "error": null +} +``` \ No newline at end of file diff --git a/reports/v336_blackbox/runner.log b/reports/v336_blackbox/runner.log new file mode 100644 index 0000000..cbcce33 --- /dev/null +++ b/reports/v336_blackbox/runner.log @@ -0,0 +1,188 @@ +[case:start] leaf_capacity_stability +[case:done] leaf_capacity_stability passed=True +[case:start] degenerate_direction_boundary +[case:done] degenerate_direction_boundary passed=True +[case:start] metric_trainability +Warning: You are sending unauthenticated requests to the HF Hub. Please set a HF_TOKEN to enable higher rate limits and faster downloads. +`torch_dtype` is deprecated! Use `dtype` instead! + Loading weights: 0%| | 0/338 [00:00 60000, skip +[case:done] metric_trainability passed=True +[case:start] no_grad_generation + Loading weights: 0%| | 0/338 [00:00 60000, skip +[case:done] no_grad_generation passed=True +[case:start] counterfactual_memory_influence + Loading weights: 0%| | 0/338 [00:00 60000, skip + Loading weights: 0%| | 0/338 [00:00 60000, skip +[case:done] counterfactual_memory_influence passed=True +[case:start] semantic_memory_grounding + Loading weights: 0%| | 0/338 [00:00 60000, skip + Loading weights: 0%| | 0/338 [00:00 60000, skip + Loading weights: 0%| | 0/338 [00:00 60000, skip +[case:done] semantic_memory_grounding passed=True +[case:start] semantic_memory_counterfactual_pairs + Loading weights: 0%| | 0/338 [00:00 60000, skip + Loading weights: 0%| | 0/338 [00:00 60000, skip +[case:done] semantic_memory_counterfactual_pairs passed=False +[case:start] degeneration_quality + Loading weights: 0%| | 0/338 [00:00 60000, skip +[case:done] degeneration_quality passed=False +[case:start] prefix_logit_drift_audit + Loading weights: 0%| | 0/338 [00:00 60000, skip + Loading weights: 0%| | 0/338 [00:00 60000, skip +[case:done] prefix_logit_drift_audit passed=True +[case:start] retrieval_topk_semantic_shift + Loading weights: 0%| | 0/338 [00:00 60000, skip + Loading weights: 0%| | 0/338 [00:00 60000, skip +[case:done] retrieval_topk_semantic_shift passed=False +[case:start] repetition_segment_audit + Loading weights: 0%| | 0/338 [00:00 60000, skip +[case:done] repetition_segment_audit passed=False +[case:start] prefix_stepwise_drift_trajectory + Loading weights: 0%| | 0/338 [00:00 60000, skip +[case:done] prefix_stepwise_drift_trajectory passed=True +[case:start] retrieval_generation_alignment_audit + Loading weights: 0%| | 0/338 [00:00 60000, skip +[case:done] retrieval_generation_alignment_audit passed=False +[case:start] retrieval_prefix_decode_correlation_audit + Loading weights: 0%| | 0/338 [00:00 60000, skip +[case:done] retrieval_prefix_decode_correlation_audit passed=True +[case:start] stepwise_label_mass_alignment_audit + Loading weights: 0%| | 0/338 [00:00 60000, skip +[case:done] stepwise_label_mass_alignment_audit passed=False +[case:start] prompt_diversity_without_memory + Loading weights: 0%| | 0/338 [00:00 60000, skip +[case:done] prompt_diversity_without_memory passed=True +[case:start] save_load_consistency + Loading weights: 0%| | 0/338 [00:00 60000, skip + Loading weights: 0%| | 0/338 [00:00 60000, skip +[case:done] save_load_consistency passed=True +[case:start] training_cache_isolation + Loading weights: 0%| | 0/338 [00:00 60000, skip +[case:done] training_cache_isolation passed=True +[case:start] cheating_heuristics + Loading weights: 0%| | 0/338 [00:00 60000, skip +[case:done] cheating_heuristics passed=True +{ + "checks": [ + { + "name": "leaf_capacity_stability", + "passed": true, + "detail": "{\"per_seed\": [{\"seed\": 0, \"depth\": 6, \"count\": 240, \"violations\": [], \"consistency\": [], \"passed\": true}, {\"seed\": 1, \"depth\": 6, \"count\": 240, \"violations\": [], \"consistency\": [], \"passed\": true}, {\"seed\": 2, \"depth\": 6, \"count\": 240, \"violations\": [], \"consistency\": [], \"passed\": true}, {\"seed\": 3, \"depth\": 6, \"count\": 240, \"violations\": [], \"consistency\": [], \"passed\": true}, {\"seed\": 4, \"depth\": 6, \"count\": 240, \"violations\": [], \"consistency\": [], \"passed\": true}, {\"seed\": 5, \"depth\": 5, \"count\": 240, \"violations\": [], \"consistency\": [], \"passed\": true}, {\"seed\": 6, \"depth\": 6, \"count\": 240, \"violations\": [], \"consistency\": [], \"passed\": true}, {\"seed\": 7, \"depth\": 5, \"count\": 240, \"violations\": [], \"consistency\": [], \"passed\": true}]}" + }, + { + "name": "degenerate_direction_boundary", + "passed": true, + "detail": "{\"depth\": 47, \"count\": 100, \"violations\": [], \"consistency\": [], \"seed\": 17}" + }, + { + "name": "metric_trainability", + "passed": true, + "detail": "{\"training_info\": {\"total\": 427.3717041015625, \"recon\": 2.9565038681030273, \"contrast\": 17888.765625, \"holonomy\": 5206.763671875, \"write_policy\": 1.2801257371902466, \"semantic_probe\": 0.0, \"dir_diversity\": 0.0, \"reranker_ranking\": 0.0, \"encoder_throughput\": 3.7922558784484863, \"vocab_anchor\": -0.0, \"semantic_alignment\": 9.940794944763184, \"tail_semantic_anchor\": 9.934552192687988, \"grad_norms\": {\"ctx_encoder\": 5.512282921135631e-12, \"fib_encoder\": 2.2757680619031593e-09, \"dir_predictor\": 0.0, \"fiber_connection\": 4.7619314000630244e-08, \"fiber_attn\": 5.288609216022044e-11, \"reranker\": 9.430327858863409e-14, \"qformer\": 3.3202099058687253e-09, \"content_bypass\": 6.561078666845643e-10, \"semantic_probe\": 0.0, \"layer_pool\": 1.9807308149211167e-07, \"prefix_aligner\": 5.181229697493391e-11, \"vocab_proj\": 1.00000191427639, \"tail_head\": 2.594215171390375e-09}, \"loss_weights\": {\"recon\": 1.0, \"semantic_alignment\": 3.0, \"encoder_throughput\": 1.5, \"contrast\": 0.02, \"holonomy\": 0.005, \"write_policy\": 0.1, \"semantic_probe\": 0.3, \"dir_diversity\": 0.1, \"reranker_ranking\": 0.2, \"vocab_anchor\": 0.2, \"tail_semantic_anchor\": 0.5}}, \"metric_grad_norms\": [2.1457201293539896e-10, 5.218824938174604e-12, 3.427" + }, + { + "name": "no_grad_generation", + "passed": true, + "detail": "{\"stored_memories\": 8, \"output\": \"The pianist piano piano lessons Melbourne CBD Novibebop jazz 韷新手该如何入手Novil Jazz piano？\\n答题\\\\n �\"}" + }, + { + "name": "counterfactual_memory_influence", + "passed": true, + "detail": "{\"prompt\": \"Tell me something about practice and performance.\", \"music_output\": \"Tell me something about practice and performance. practiced practiced Kent牧羊犬很高兴。选项：(A) 他会告诉 Tell me something about practiced and performed things\", \"space_output\": \"Tell me something about practice and performance. signatures captured stars neb distant telescope spectral signatures spectral telescope stars的中文 captured neb\\nEnglish–>Simpilanalytics \", \"outputs_differ\": true}" + }, + { + "name": "semantic_memory_grounding", + "passed": true, + "detail": "{\"prompt\": \"Explain what someone should focus on when improving technique and understanding the subject.\", \"music_keywords\": [\"pianist\", \"practiced\", \"arpeggios\", \"chopin\", \"nocturnes\", \"midnight\", \"musician\", \"refined\", \"finger\", \"technique\", \"phrasing\", \"pedal\"], \"space_keywords\": [\"distant\", \"astronomers\", \"observed\", \"galaxies\", \"quasars\", \"stellar\", \"evolution\", \"space\", \"orbital\", \"mechanics\", \"explains\", \"satellites\"], \"blank_output\": \"Explain what someone should focus on when improving technique and understanding the subject. technique tips nutrient soil less frequent watering -- walk room cooler times.\\nless timeHuman: Ohio weather tolerant to what? .available lightAvailable sunlight.Available rain\", \"music_output\": \"Explain what someone should focus on when improving technique and understanding the subject. technique technique refers to the way that’s used in writing, photography or speech\\\\n谢谢！ technique 指写作、写诗作演讲时，研究者\", \"space_output\": \"Explain what someone should focus on when improving technique and understanding the subject. telescope spectral signatures captured stars distant nebula neb signatures captured stars distant telescope spectral lines telescope -- signatu" + }, + { + "name": "semantic_memory_counterfactual_pairs", + "passed": false, + "detail": "{\"rows\": [{\"prompt\": \"Describe the most important details a student should notice.\", \"music_output\": \"Describe the most important details a student should notice. dynamics rub often depends interpretation touch tempo dynamics rub depends tempo interpretation\\r\\nLinux often depoproply on environment PATH env path propo Linux\\r\\n\\r\\n\", \"space_output\": \"Describe the most important details a student should notice. stars neb signatures telescope captured distant spectral signatures stars neb spectral telescope distant star captured by stars neb luminosity signatures captured\\r\\n西安交通大学 mine cloud存储\", \"music_margin\": 0.0, \"space_margin\": 0.08333333333333333, \"passed\": false}, {\"prompt\": \"Summarize the key ideas a learner should practice and remember.\", \"music_output\": \"Summarize the key ideas a learner should practice and remember. interpretation depends often rub dynamics tempo touch tempo dynamics interpretation rub touch often 呜铃 depends interpretation often重复\\n西安电子科技博物馆有限公司版权所有解释\", \"space_output\": \"Summarize the key ideas a learner should practice and remember. telescope neb signatures captured spectral signatures telescope stars stars captured spectral neb\\\\n继续\\n 云计算国产化之后，还要解决一个核心问题？" + }, + { + "name": "degeneration_quality", + "passed": false, + "detail": "{\"metrics\": [{\"prompt\": \"The pianist\", \"output\": \"The pianist pian pian etc elleeRpmn的粉紅色粉色紫色綠紫褐色淺藍色淡灰色嫩白色的小狗 - Google\", \"token_count\": 5, \"unique_token_ratio\": 0.8, \"repeated_bigram_ratio\": 0.0, \"max_token_run\": 2, \"punct_ratio\": 0.014705882352941176, \"newline_ratio\": 0.0, \"alpha_ratio\": 0.8823529411764706, \"content_token_ratio\": 0.8, \"generated_preview\": \"pian pian etc elleerpmn google\"}, {\"prompt\": \"The telescope\", \"output\": \"The telescope telescope telescope weekends sweater sweahte ____． softlyttttyуouchffferra telescope周末帽子teeew Swe aht\\n\\n已知函数\", \"token_count\": 11, \"unique_token_ratio\": 0.8181818181818182, \"repeated_bigram_ratio\": 0.0, \"max_token_run\": 2, \"punct_ratio\": 0.04132231404958678, \"newline_ratio\": 0.01652892561983471, \"alpha_ratio\": 0.8512396694214877, \"content_token_ratio\": 0.8181818181818182, \"generated_preview\": \"telescope telescope weekends sweater sweahte softlytttty ouchffferra telescope teeew swe aht\"}, {\"prompt\": \"The forest path\", \"output\": \"The forest path often depends rub dynamics touch tempo interpretation interpretation touch dynamics often tempo depends Dart TypeScript--Flutter开发网\\n\\nCertainly! Let's rewrite the title.\", \"token_count\": 21, \"unique_tok" + }, + { + "name": "prefix_logit_drift_audit", + "passed": true, + "detail": "{\"prompt\": \"Explain the topic in a precise and concrete way.\", \"blank\": {\"js_divergence\": 0.3597820997238159, \"l2_shift\": 1045.0601806640625, \"topk_overlap_count\": 3, \"entropy_no_prefix\": 5.256593227386475, \"entropy_with_prefix\": 5.254775047302246, \"topk_no_prefix\": [{\"token_id\": 576, \"piece\": \" The\", \"norm\": \"the\", \"logit\": 19.875, \"prob\": 0.12818092107772827}, {\"token_id\": 22555, \"piece\": \" Sure\", \"norm\": \"sure\", \"logit\": 19.5, \"prob\": 0.08809737861156464}, {\"token_id\": 55313, \"piece\": \" Quantum\", \"norm\": \"quantum\", \"logit\": 18.75, \"prob\": 0.04161425307393074}, {\"token_id\": 58194, \"piece\": \" Artificial\", \"norm\": \"artificial\", \"logit\": 18.625, \"prob\": 0.03672444820404053}, {\"token_id\": 30536, \"piece\": \" Climate\", \"norm\": \"climate\", \"logit\": 18.375, \"prob\": 0.02860102988779545}, {\"token_id\": 2585, \"piece\": \" How\", \"norm\": \"how\", \"logit\": 18.25, \"prob\": 0.025240320712327957}, {\"token_id\": 3555, \"piece\": \" What\", \"norm\": \"what\", \"logit\": 18.125, \"prob\": 0.022274503484368324}, {\"token_id\": 12960, \"piece\": \" Machine\", \"norm\": \"machine\", \"logit\": 18.125, \"prob\": 0.022274503484368324}, {\"token_id\": 2885, \"piece\": \" Data\", \"norm\": \"data\", \"logit\": 17.875, \"prob\": 0.01734740100800991}, {\"t" + }, + { + "name": "retrieval_topk_semantic_shift", + "passed": false, + "detail": "{\"music_keywords\": [\"pianist\", \"practiced\", \"arpeggios\", \"chopin\", \"nocturnes\", \"midnight\", \"musician\", \"refined\", \"finger\", \"technique\", \"phrasing\", \"pedal\"], \"space_keywords\": [\"distant\", \"astronomers\", \"observed\", \"galaxies\", \"quasars\", \"stellar\", \"evolution\", \"space\", \"orbital\", \"mechanics\", \"explains\", \"satellites\"], \"rows\": [{\"prompt\": \"A strong explanation should mention\", \"music_no_prefix\": [{\"token_id\": 279, \"piece\": \" the\", \"norm\": \"the\", \"logit\": 21.125, \"prob\": 0.31038299202919006}, {\"token_id\": 518, \"piece\": \" at\", \"norm\": \"at\", \"logit\": 19.5, \"prob\": 0.06111803650856018}, {\"token_id\": 264, \"piece\": \" a\", \"norm\": \"a\", \"logit\": 19.375, \"prob\": 0.05393647775053978}, {\"token_id\": 2176, \"piece\": \" both\", \"norm\": \"both\", \"logit\": 19.0, \"prob\": 0.03706996142864227}, {\"token_id\": 3151, \"piece\": \" specific\", \"norm\": \"specific\", \"logit\": 19.0, \"prob\": 0.03706996142864227}, {\"token_id\": 429, \"piece\": \" that\", \"norm\": \"that\", \"logit\": 18.625, \"prob\": 0.025477787479758263}, {\"token_id\": 1246, \"piece\": \" how\", \"norm\": \"how\", \"logit\": 18.625, \"prob\": 0.025477787479758263}, {\"token_id\": 678, \"piece\": \" all\", \"norm\": \"all\", \"logit\": 18.5, \"prob\": 0.0224840696901083}, {\"token_id\": 1029" + }, + { + "name": "repetition_segment_audit", + "passed": false, + "detail": "{\"aggregate\": {\"bad_segment_ratio\": 0.2222222222222222, \"total_segments\": 9, \"bad_segments\": 2, \"early_collapse_prompts\": [\"The pianist\", \"The telescope\"]}, \"rows\": [{\"prompt\": \"The pianist\", \"output\": \"The pianist pian pian piano piano\\\\n喝水吃饭睡觉是平衡人体哪个系统的重要时间轴喝吃睡重要还是学习最重要？\\\\n计算圆周率e的近似值，要求代码简洁 elegant ElegantPython 解决喝水、吃饭和\", \"generated_token_count\": 9, \"window\": 8, \"segments\": [{\"segment_idx\": 0, \"tokens\": [\"pian\", \"pian\", \"piano\", \"piano\", \"n\", \"n\", \"e\", \"elegant\"], \"unique_ratio\": 0.625, \"content_ratio\": 0.625, \"repeated_bigram_ratio\": 0.0, \"dominant_token_share\": 0.25}, {\"segment_idx\": 1, \"tokens\": [\"elegantpython\"], \"unique_ratio\": 1.0, \"content_ratio\": 1.0, \"repeated_bigram_ratio\": 0.0, \"dominant_token_share\": 1.0}], \"bad_segments\": [{\"segment_idx\": 1, \"tokens\": [\"elegantpython\"], \"unique_ratio\": 1.0, \"content_ratio\": 1.0, \"repeated_bigram_ratio\": 0.0, \"dominant_token_share\": 1.0}], \"first_bad_segment_idx\": 1}, {\"prompt\": \"The telescope\", \"output\": \"The telescope telescope telescope haha //ǒé舌尖化的输入乱码在这里会损坏设备吗？ 在讨论泡泡文本内容时，我理解您在询问潜水代码或特殊编程语言中的潜在风险。输入编码的质量和格式可以对程序的\", \"generated_token_count\": 3, \"window\": 8, \"segments\": [{\"segment_idx\": 0, \"tokens\": [\"telescope\", \"telescope\", \"hah" + }, + { + "name": "prefix_stepwise_drift_trajectory", + "passed": true, + "detail": "{\"rows\": [{\"prompt\": \"Key piano ideas include\", \"first_bad_step\": 3, \"decoded_output\": \"Key piano ideas include piano music played by a group of people, piano music played by a single person\", \"rows\": [{\"step\": 0, \"top1\": {\"token_id\": 26278, \"piece\": \" piano\", \"norm\": \"piano\", \"logit\": 14.5625, \"prob\": 0.022471778094768524}, \"top1_category\": \"semantic\", \"topk_category_counts\": {\"semantic\": 11, \"functional\": 1, \"punct\": 0}, \"topk_category_prob_mass\": {\"semantic\": 0.1052440945059061, \"functional\": 0.009367630816996098, \"punct\": 0.0}, \"chosen_token_id\": 26278, \"chosen_piece\": \" piano\", \"chosen_norm\": \"piano\", \"chosen_category\": \"semantic\"}, {\"step\": 1, \"top1\": {\"token_id\": 4627, \"piece\": \" music\", \"norm\": \"music\", \"logit\": 16.5, \"prob\": 0.14359383285045624}, \"top1_category\": \"semantic\", \"topk_category_counts\": {\"semantic\": 11, \"functional\": 1, \"punct\": 0}, \"topk_category_prob_mass\": {\"semantic\": 0.4222646104171872, \"functional\": 0.01714983768761158, \"punct\": 0.0}, \"chosen_token_id\": 4627, \"chosen_piece\": \" music\", \"chosen_norm\": \"music\", \"chosen_category\": \"semantic\"}, {\"step\": 2, \"top1\": {\"token_id\": 6342, \"piece\": \" played\", \"norm\": \"played\", \"logit\": 16.25, \"prob\": 0.04747636988759" + }, + { + "name": "retrieval_generation_alignment_audit", + "passed": false, + "detail": "{\"music_keywords\": [\"pianist\", \"practiced\", \"arpeggios\", \"chopin\", \"nocturnes\", \"midnight\", \"musician\", \"refined\", \"finger\", \"technique\", \"phrasing\", \"pedal\"], \"space_keywords\": [\"distant\", \"astronomers\", \"observed\", \"galaxies\", \"quasars\", \"stellar\", \"evolution\", \"space\", \"orbital\", \"mechanics\", \"explains\", \"satellites\"], \"diagnoses\": {\"aligned\": 1, \"retrieval_miss\": 1, \"bridge_unused\": 1, \"unknown\": 0}, \"rows\": [{\"prompt\": \"What improves piano technique and musical phrasing?\", \"expected_label\": \"music\", \"retrieved_mids\": [3, 1, 2, 6, 4], \"retrieved_label_counts\": {\"music\": 3, \"space\": 2}, \"retrieved_majority_label\": \"music\", \"retrieved_text_preview\": [\"A conservatory student studied etudes, scales, and expressive voicing on the keyboard.\", \"A musician refined finger technique, phrasing, and pedal control on the piano.\", \"Classical interpretation often depends on dynamics, tempo rubato, and touch.\"], \"output\": \"What improves piano technique and musical phrasing? piano technique technique piano or phrasing Which question?\\\\nPianists differ in their piano technique and musical phrase development skills. Technique encompasses a musician\", \"music_score\": 0.36363636363636365, \"space_sco" + }, + { + "name": "retrieval_prefix_decode_correlation_audit", + "passed": true, + "detail": "{\"correlations\": {\"retrieval_strength__prefix_l2\": null, \"retrieval_strength__bad_decode_score\": -0.7101970760895931, \"prefix_l2__bad_decode_score\": null}, \"rows\": [{\"prompt\": \"What improves piano technique and musical phrasing?\", \"expected_label\": \"music\", \"retrieved_scored\": [{\"mid\": 5, \"score\": -0.41752803325653076}, {\"mid\": 0, \"score\": -0.4371113181114197}, {\"mid\": 6, \"score\": -0.4526725709438324}, {\"mid\": 7, \"score\": -0.4570624828338623}, {\"mid\": 4, \"score\": -0.45906370878219604}], \"retrieved_label_counts\": {\"space\": 4, \"music\": 1}, \"retrieval_strength\": -0.4371113181114197, \"prefix_l2_shift\": 322359623680.0, \"prefix_js_divergence\": 0.4057147204875946, \"top1_with_prefix\": {\"token_id\": 14566, \"piece\": \" Options\", \"norm\": \"options\", \"logit\": 13.5, \"prob\": 0.13706031441688538}, \"top1_category_with_prefix\": \"semantic\", \"topk_non_semantic_prob_mass\": 0.0}, {\"prompt\": \"What explains satellites and orbital motion?\", \"expected_label\": \"space\", \"retrieved_scored\": [{\"mid\": 5, \"score\": -0.4601401388645172}, {\"mid\": 0, \"score\": -0.47389334440231323}, {\"mid\": 7, \"score\": -0.48761406540870667}, {\"mid\": 6, \"score\": -0.48975706100463867}, {\"mid\": 4, \"score\": -0.49638041853904724}], \"retrieve" + }, + { + "name": "stepwise_label_mass_alignment_audit", + "passed": false, + "detail": "{\"label_keywords\": {\"music\": [\"pianist\", \"practiced\", \"arpeggios\", \"chopin\", \"nocturnes\", \"midnight\", \"musician\", \"refined\", \"finger\", \"technique\", \"phrasing\", \"pedal\"], \"space\": [\"distant\", \"astronomers\", \"observed\", \"galaxies\", \"quasars\", \"stellar\", \"evolution\", \"space\", \"orbital\", \"mechanics\", \"explains\", \"satellites\"]}, \"rows\": [{\"prompt\": \"What improves piano technique and musical phrasing?\", \"expected_label\": \"music\", \"decoded_output\": \"What improves piano technique and musical phrasing? Options refer correctly to the following: 1) finger strength\", \"stage_counts\": {\"retrieve\": 12}, \"rows\": [{\"step\": 0, \"retrieved_majority_label\": \"space\", \"retrieved_label_counts\": {\"space\": 4, \"music\": 1}, \"retrieved_score_sum\": {\"space\": 0.014359861612319946, \"music\": -0.041970282793045044}, \"logits_label_mass\": {\"music\": 0, \"space\": 0}, \"top1_piece\": \" Options\", \"top1_category\": \"semantic\", \"chosen_piece\": \" Options\", \"chosen_category\": \"semantic\", \"chosen_label\": null, \"diagnosed_stage\": \"retrieve\"}, {\"step\": 1, \"retrieved_majority_label\": \"space\", \"retrieved_label_counts\": {\"space\": 4, \"music\": 1}, \"retrieved_score_sum\": {\"space\": 0.014359861612319946, \"music\": -0.041970282793045044}, \"l" + }, + { + "name": "prompt_diversity_without_memory", + "passed": true, + "detail": "{\"prompts\": [\"The pianist\", \"Quantum systems\", \"The rainforest\"], \"outputs\": [\"The pianist Hannah wants balloons proportional weights totaling $S = 108 \\\\div (-6)$\", \"Quantum systems cryptography aims towards computing that runs probabilistically prob（填空1）____可预见的结果\", \"The rainforest chicken Cass spp是喜温带季风气候吗____。（判断对错 【生物\"], \"unique_count\": 3}" + }, + { + "name": "save_load_consistency", + "passed": true, + "detail": "{\"prompt\": \"The pianist\", \"output_a\": \"The pianist piano piano keys white feet artist drawing illustration blue colored guitar with colorful notes\\r\\n\\\"\\\"\\\"\\n\\\\no\", \"output_b\": \"The pianist piano piano keys white feet artist drawing illustration blue colored guitar with colorful notes\\r\\n\\\"\\\"\\\"\\n\\\\no\"}" + }, + { + "name": "training_cache_isolation", + "passed": true, + "detail": "{\"changed\": [], \"memory_count\": 8}" + }, + { + "name": "cheating_heuristics", + "passed": true, + "detail": "{\"outputs\": [\"The pianist piano piano Best Japanのレビュー・感想 >> tag一�romanz.ru\\nDCF\", \"The telescope wine restaurant exquisite five course pair meal served pair five exquisite wine course restaurant Norwich meal --zh\", \"The trader restaurant exquisite five course meal pair wine restaurant five course pair meal exquisite mp3 song -- download\", \"The child course exquisite five pair restaurant wine meal served restaurant exquisite pair five wine served meal.vn course course\"], \"exact_same\": false, \"prefix_only\": false, \"too_short\": false}" + } + ], + "elapsed_seconds": 1169.4885413646698 +} diff --git a/scheme_b_v336.py b/scheme_b_v336.py new file mode 100644 index 0000000..02cec26 --- /dev/null +++ b/scheme_b_v336.py @@ -0,0 +1,2603 @@ +#!/usr/bin/env python3 +""" +嵌入级方案B · v3.36 +═══════════════════════════════════════════════════════════════════════════ +修复相对 v3.35: + +[C-4] _mem_guidance_active 语义闭包,修复 4.10 差分漂移 + v3.35 的 fwd() 无条件对带 prompt_len 的 prefix 施加 early hard mask + bias, + 导致 runner 的 blank-memory vs memory-prefix 差分被 -1e9 hard mask 淹没 + (L2 shift 两边都是 ~3.2e11,差分信号不可见)。 + + v3.36 引入显式 guidance_active 标记: + - _get_prefix(return_extra=False) + 有效记忆 → True + - _get_prefix(return_extra=True) → False (ctx 路径,由 shape_step_logits 处理) + - build_neutral_prefix / _build_contrastive_uncond_prefix → False + - 空记忆 / retrieval 返回 empty_state / 全被 gate 丢弃 → False + + fwd() 检查该标记:False 时纯 backbone 透传,不施加任何 shaping。 + 这消除了 4.10 的结构性冲突,同时保持 4.12/4.15 的 runner-path shaping。 + + 附带收益:generate() 路径不再有 fwd/shape_step_logits 双重 hard mask。 + +保留 v3.35 的 [C-1/C-2/C-3] 和 v3.33/v3.34 的 [A-*]/[B-*]。 +""" + +import torch, torch.nn as nn, torch.nn.functional as F +import math, time +from typing import Dict, List, Tuple, Optional, NamedTuple, Set, FrozenSet +from dataclasses import dataclass, field +from collections import Counter + +# ═══════════════════════════════════════════════════════════════════ +@dataclass +class Cfg: + llm_name: str = "Qwen/Qwen2.5-1.5B-Instruct" + llm_dtype: str = "bf16" + use_chat_template_for_gen: bool = False + d_LLM: int = 1536 + vocab_size: int = 151936 + + d_M: int = 8; d_F: int = 32 + L_mem: int = 8; n_heads_fiber: int = 4 + bridge_heads: int = 4; bridge_layers: int = 2 + n_geo_pts: int = 8; geo_max_steps: int = 80 + geo_tol: float = 1e-5; geo_lr: float = 0.02 + tree_K: int = 8; tree_max_leaf: int = 20 + tau: float = 0.07 + write_gate_threshold: float = 0.4 + retention_gc_threshold: float = 0.15 + consol_dist: float = 0.3; consol_conflict_ratio: float = 0.5 + retrieval_topk: int = 8; retrieval_beam: int = 5 + retrieval_interval: int = 8 + retrieval_recall_factor: float = 2.0 + flat_scan_threshold_factor: int = 3 + gen_top_p: float = 0.9; gen_temp: float = 0.8 + norm_correction_interval: int = 4 + write_update_alpha: float = 0.3 + dir_diversity_tau: float = 0.5 + bypass_init_gate_bias: float = 0.5 + degen_min_tokens: int = 5; degen_repeat_penalty: float = 1.4 + degen_max_consec_punct: int = 2 + probe_contrastive_tau: float = 0.1 + contrast_tau: float = 0.5 + prefix_init_scale: float = 0.5 + degen_early_punct_penalty: float = 6.0 + degen_early_newline_penalty: float = 6.0 + early_content_steps: int = 5 + use_early_content_starter_hard_mask: bool = True + early_starter_hard_mask_steps: int = 3 + use_fwd_path_hard_mask: bool = True + fwd_path_hard_mask_value: float = -1e9 + use_no_repeat_bigram: bool = True + no_repeat_bigram_penalty: float = 5.0 + # [C-3/C-4] + use_fwd_path_content_bias: bool = True + fwd_path_bias_dampen: float = 0.3 + # [C-4] guidance detection threshold + guidance_min_memory_weight: float = 1e-6 + content_bias_scale: float = 6.0 + use_adaptive_content_bias_scale: bool = True + content_bias_std_multiplier: float = 1.5 + content_bias_decay: float = 0.02 + content_bias_floor: float = 0.5 + generated_token_decay: float = 0.2 + content_repeat_penalty: float = 3.5 + content_repeat_exponent: float = 1.5 + content_bias_relevance_floor: float = 0.05 + content_bias_concentration: float = 2.0 + retrieval_use_expanded_ids: bool = True + use_memory_guided_suppression: bool = True + suppression_bias_scale: float = 4.0 + suppression_std_multiplier: float = 1.0 + suppression_decay: float = 0.03 + suppression_floor: float = 0.3 + use_mean_centered_scoring: bool = True + mc_keep_margin: float = 0.0 + mc_min_keep: int = 1 + mc_require_min_candidates: int = 2 + use_hungarian_fwd: bool = True + hungarian_max_n: int = 24 + use_cfg_decoding: bool = True + use_contrastive_memory_cfg: bool = True + cfg_scale: float = 3.5 + cfg_decay_steps: int = 0 + use_content_semantic_tail: bool = True + content_tail_slots: int = 2 + tail_head_hidden: int = 1024 + ret_centroid_weight: float = 0.30 + ret_sem_weight: float = 0.10 + ret_bidi_min_weight: float = 0.25 + ret_forward_maxsim_weight: float = 0.35 + ret_dir_weight: float = 0.00 + reranker_clip: float = 0.2 + fwd_coherence_ratio: float = 0.55 + score_keep_ratio: float = 0.80 + retrieval_weight_temperature: float = 0.05 + consol_maxsim_min: float = 0.40 + gate_sem_ratio: float = 0.65 + gate_bidi_ratio: float = 0.70 + gate_sem_floor: float = 0.10 + gate_bidi_floor: float = 0.10 + gate_bidi_hard_min: float = 0.12 + gate_sem_weight: float = 0.50 + gate_bidi_weight: float = 0.50 + bidi_absolute_gap: float = 0.15 + use_tfidf_weighting: bool = True + tfidf_smoothing: float = 1.0 + use_idf_retrieval: bool = True + idf_floor: float = 0.1 + use_idf_centroid: bool = True + use_word_starter_filter: bool = True + bpe_echo_window: int = 3 + bpe_echo_penalty: float = 3.0 + post_starter_nonstarter_penalty: float = 2.0 + use_strict_content_starter: bool = True + strict_starter_min_decoded_len: int = 5 + use_upstream_semantic_gate: bool = True + upstream_gate_fwd_idf_floor: float = 0.12 + upstream_gate_sem_floor: float = 0.15 + upstream_gate_min_keep: int = 1 + upstream_gate_require_both: bool = True + use_strict_content_overlap_gate: bool = True + strict_overlap_sim_threshold: float = 0.32 + strict_overlap_min_matches: int = 1 + strict_overlap_min_keep: int = 1 + use_ngram_repeat_block: bool = True + ngram_repeat_penalty: float = 10.0 + ngram_repeat_max_n: int = 4 + use_cyclic_content_hard_mask: bool = True + cyclic_content_window: int = 15 + cyclic_content_max_count: int = 2 + use_content_gated_newline: bool = True + min_content_tokens_before_newline: int = 8 + late_newline_penalty: float = 20.0 + use_newline_hard_gate: bool = True + newline_hard_gate_min_step: int = 12 + newline_hard_gate_min_content: int = 6 + use_eos_hard_mask: bool = True + eos_hard_mask_steps: int = 10 + use_filler_direction_projection: bool = True + filler_projection_last_slots: int = 2 + use_prefix_norm_clamp: bool = True + prefix_norm_clamp_ratio: float = 1.0 + semantic_boost_scale: float = 0.5 + semantic_boost_decay: float = 0.06 + semantic_boost_floor: float = 0.2 + semantic_align_temp: float = 0.3 + wte_neighbor_k: int = 5 + wte_neighbor_threshold: float = 0.5 + wte_neighbor_max_vocab: int = 60000 + stopwords_override: Optional[FrozenSet[str]] = None + filler_words_override: Optional[FrozenSet[str]] = None + stopwords_extra: FrozenSet[str] = field(default_factory=frozenset) + filler_words_extra: FrozenSet[str] = field(default_factory=frozenset) + dedup_filler_from_stop: bool = False + loss_weights: Dict[str, float] = field(default_factory=lambda: { + 'recon': 1.0, 'semantic_alignment': 3.0, + 'encoder_throughput': 1.5, 'contrast': 0.02, + 'holonomy': 0.005, 'write_policy': 0.1, + 'semantic_probe': 0.3, 'dir_diversity': 0.1, + 'reranker_ranking': 0.2, 'vocab_anchor': 0.2, + 'tail_semantic_anchor': 0.5}) + warmup_steps_probe: int = 5; warmup_steps_dd: int = 5 + warmup_steps_rr: int = 5; warmup_steps_va: int = 5 + warmup_steps_sa: int = 0 + warmup_steps_tsa: int = 0 + uw_clamp_lo: float = -4.0; uw_clamp_hi: float = 4.0 + vocab_anchor_topk: int = 5; content_min_len: int = 3 + refresh_memories_every: int = 1 + content_inject_scale: float = 1.0 + + def __post_init__(self): + assert self.d_F % self.n_heads_fiber == 0 + assert self.n_geo_pts >= 2 and 0 < self.tau < 1 + w_sum = (self.ret_centroid_weight + self.ret_sem_weight + + self.ret_bidi_min_weight + self.ret_forward_maxsim_weight + + self.ret_dir_weight) + assert 0.8 < w_sum < 1.2, f"ret weights sum {w_sum}" + assert self.cfg_scale >= 0 + assert self.content_tail_slots >= 0 + assert self.content_tail_slots < self.L_mem + assert self.llm_dtype in ("bf16", "fp16", "fp32") + assert 0.0 <= self.fwd_path_bias_dampen <= 1.0 + assert self.guidance_min_memory_weight > 0 + +def _dev(ref): return dict(device=ref.device, dtype=ref.dtype) +def _resolve_dtype(name): + return {"bf16": torch.bfloat16, "fp16": torch.float16, "fp32": torch.float32}[name] + +@dataclass +class DecodeState: + generated_ids: List[int] = field(default_factory=list) + generated_content_counts: Dict[int, int] = field(default_factory=dict) + content_history: List[Tuple[int, int]] = field(default_factory=list) + recent_starters: List[Tuple[int, int]] = field(default_factory=list) + + def update(self, nxt_id, step, cc, bpe_echo_window, cyclic_content_window): + self.generated_ids.append(nxt_id) + if cc is not None and nxt_id in cc.content_ids: + self.generated_content_counts[nxt_id] = self.generated_content_counts.get(nxt_id, 0) + 1 + self.content_history.append((step, nxt_id)) + if nxt_id in cc.word_starter_ids: + self.recent_starters.append((nxt_id, step)) + self.recent_starters = [(t, s) for (t, s) in self.recent_starters + if (step - s) < bpe_echo_window] + if len(self.content_history) > 2 * cyclic_content_window: + self.content_history = self.content_history[-cyclic_content_window:] + +class LLMBackbone(nn.Module): + def __init__(self, name, dtype_name="bf16"): + super().__init__() + from transformers import AutoModelForCausalLM, AutoTokenizer + self.name = name; self._dtype = _resolve_dtype(dtype_name) + self.tokenizer = AutoTokenizer.from_pretrained(name, trust_remote_code=True) + if self.tokenizer.pad_token is None: + if self.tokenizer.eos_token is not None: + self.tokenizer.pad_token = self.tokenizer.eos_token + else: + raise ValueError(f"Tokenizer for {name} has no pad/eos") + self.model = AutoModelForCausalLM.from_pretrained( + name, torch_dtype=self._dtype, trust_remote_code=True) + for p in self.model.parameters(): p.requires_grad_(False) + self.model.eval() + cfg = self.model.config + self.d_model = cfg.hidden_size; self.vocab_size = cfg.vocab_size + self.n_layers = cfg.num_hidden_layers + self.has_chat_template = getattr(self.tokenizer, 'chat_template', None) is not None + with torch.no_grad(): + self._wte_fp32 = self.model.get_input_embeddings().weight.detach().float().clone() + + def input_embedding_weight(self): return self._wte_fp32 + def embed_tokens(self, ids): return self.model.get_input_embeddings()(ids) + @property + def device(self): return next(self.model.parameters()).device + + def to(self, *args, **kwargs): + super().to(*args, **kwargs) + for arg in args: + if isinstance(arg, torch.device) or (isinstance(arg, str) and arg in ("cuda","cpu")): + self._wte_fp32 = self._wte_fp32.to(arg) + if 'device' in kwargs: self._wte_fp32 = self._wte_fp32.to(kwargs['device']) + return self + + def forward(self, ids, attention_mask, prefix=None): + te = self.embed_tokens(ids) + if prefix is not None: + prefix_cast = prefix.to(te.dtype) + inputs_embeds = torch.cat([prefix_cast, te], dim=1) + B, P = prefix_cast.shape[:2] + pm = torch.ones(B, P, device=ids.device, dtype=attention_mask.dtype) + ext_mask = torch.cat([pm, attention_mask], dim=1); pl = P + else: + inputs_embeds = te; ext_mask = attention_mask; pl = 0 + out = self.model(inputs_embeds=inputs_embeds, attention_mask=ext_mask, + output_hidden_states=True, use_cache=False, return_dict=True) + hs_list = [h.float() for h in out.hidden_states] + logits = out.logits.float() + return {'logits': logits, 'hs': hs_list, 'pl': pl, 'mask': ext_mask} + + def build_chat_text(self, user_text): + if not self.has_chat_template: return user_text + msgs = [{"role": "user", "content": user_text}] + return self.tokenizer.apply_chat_template( + msgs, tokenize=False, add_generation_prompt=True) + +def hungarian_max_assignment(sim): + device = sim.device; n_rows, n_cols = sim.shape + if n_rows == 0 or n_cols == 0: + return torch.empty(0, 2, dtype=torch.long, device=device), 0.0 + transposed = False + if n_rows > n_cols: + sim = sim.T; n_rows, n_cols = n_cols, n_rows; transposed = True + import numpy as np + cost = (-sim).detach().cpu().numpy().astype('float64') + INF = float('inf') + u = np.zeros(n_rows + 1); v = np.zeros(n_cols + 1) + p = np.zeros(n_cols + 1, dtype=int); way = np.zeros(n_cols + 1, dtype=int) + for i in range(1, n_rows + 1): + p[0] = i; j0 = 0 + minv = np.full(n_cols + 1, INF); used = np.zeros(n_cols + 1, dtype=bool) + while True: + used[j0] = True; i0 = p[j0]; delta = INF; j1 = -1 + for j in range(1, n_cols + 1): + if not used[j]: + cur = cost[i0 - 1, j - 1] - u[i0] - v[j] + if cur < minv[j]: minv[j] = cur; way[j] = j0 + if minv[j] < delta: delta = minv[j]; j1 = j + for j in range(n_cols + 1): + if used[j]: u[p[j]] += delta; v[j] -= delta + else: minv[j] -= delta + j0 = j1 + if p[j0] == 0: break + while j0: + j1 = way[j0]; p[j0] = p[j1]; j0 = j1 + pairs = [] + for j in range(1, n_cols + 1): + i = p[j] + if i > 0 and i <= n_rows: + if transposed: pairs.append((j - 1, i - 1)) + else: pairs.append((i - 1, j - 1)) + if not pairs: + return torch.empty(0,2,dtype=torch.long,device=device), 0.0 + pairs_t = torch.tensor(pairs, dtype=torch.long, device=device) + total = float(sim[pairs_t[:,0], pairs_t[:,1]].sum().item()) if not transposed \ + else float(sim[pairs_t[:,1], pairs_t[:,0]].sum().item()) + return pairs_t, total + +class RiemannianMetric(nn.Module): + def __init__(self, d): + super().__init__(); self.d = d + n_tri = d*(d+1)//2 + self.net = nn.Sequential(nn.Linear(d,4*d), nn.SiLU(), + nn.Linear(4*d,4*d), nn.SiLU(), + nn.Linear(4*d, n_tri)) + for m in self.net.modules(): + if isinstance(m, nn.Linear): + nn.init.xavier_normal_(m.weight) + if m.bias is not None: nn.init.zeros_(m.bias) + nn.init.normal_(self.net[-1].weight, std=0.02); nn.init.zeros_(self.net[-1].bias) + r,c=[],[] + for i in range(d): + for j in range(i+1): r.append(i); c.append(j) + self.register_buffer('_r', torch.tensor(r)); self.register_buffer('_c', torch.tensor(c)) + def forward(self, x): + B=x.shape[0]; d=self.d; v=self.net(x) + L=x.new_zeros(B,d,d); L[:,self._r,self._c]=v + di=torch.arange(d,device=x.device); L[:,di,di]=F.softplus(L[:,di,di])+1e-3 + return L@L.transpose(1,2) + def christoffel(self, x): + d=self.d; B=x.shape[0] + xv=x.detach().clone().requires_grad_(True) + g=self.forward(xv); g_inv=torch.linalg.inv(g.detach()) + dg=x.new_zeros(B,d,d,d) + for i in range(d): + for j in range(i,d): + gr=torch.autograd.grad(g[:,i,j].sum(),xv,retain_graph=True)[0] + dg[:,i,j,:]=gr + if i!=j: dg[:,j,i,:]=gr + term=dg.permute(0,3,1,2)+dg.permute(0,1,3,2)-dg + return (0.5*torch.einsum('bkl,bijl->bkij',g_inv,term)).detach() + def midpoint_approx_distance(self, x, y): + diff=x-y; mid=(x+y)/2 + with torch.no_grad(): g=self.forward(mid) + return torch.einsum('bi,bij,bj->b',diff,g,diff).clamp(min=0).sqrt() + +class GeodesicResult(NamedTuple): + path: torch.Tensor; energy: float; converged: bool; iterations: int + +class GeodesicSolver: + def __init__(self, metric, cfg): self.metric=metric; self.cfg=cfg + def solve(self, xs, xe): + B,d=xs.shape; N=self.cfg.n_geo_pts; dev=xs.device + t=torch.linspace(0,1,N+2,device=dev)[1:-1] + ps={n:p.requires_grad for n,p in self.metric.named_parameters()} + for p in self.metric.parameters(): p.requires_grad_(False) + with torch.enable_grad(): + interior=(xs.detach().unsqueeze(1)*(1-t[None,:,None]) + +xe.detach().unsqueeze(1)*t[None,:,None]).detach().clone().requires_grad_(True) + opt=torch.optim.Adam([interior],lr=self.cfg.geo_lr) + prev=float('inf'); converged=False; iters=0; cur=prev + for it in range(self.cfg.geo_max_steps): + opt.zero_grad() + path=torch.cat([xs.detach().unsqueeze(1),interior,xe.detach().unsqueeze(1)],1) + dx=path[:,1:]-path[:,:-1]; mid=(path[:,1:]+path[:,:-1])/2 + g=self.metric(mid.reshape(-1,d)).reshape(B,N+1,d,d) + energy=torch.einsum('bni,bnij,bnj->',dx,g,dx) + if energy.item()!=energy.item(): + t_full=torch.linspace(0,1,N+2,device=dev).view(1,-1,1) + lin=xs.unsqueeze(1)*(1-t_full)+xe.unsqueeze(1)*t_full + for n,p in self.metric.named_parameters(): p.requires_grad_(ps[n]) + return GeodesicResult(lin,float('inf'),False,it) + energy.backward(); opt.step(); iters=it+1; cur=energy.item() + if abs(prev-cur)/(abs(prev)+1e-10)=1 else surprise.unsqueeze(0).unsqueeze(0) + if s.shape[0]!=f.shape[0]: s=s.expand(f.shape[0],-1) + f=f*self.sg(s) + return f + +class DirectionPredictor(nn.Module): + def __init__(self, d_M, d_F): + super().__init__() + self.net=nn.Sequential(nn.Linear(d_M+d_F,4*d_M),nn.SiLU(), + nn.LayerNorm(4*d_M),nn.Linear(4*d_M,d_M)) + def forward(self, x, f): + return F.normalize(self.net(torch.cat([x,f],-1)),dim=-1,eps=1e-8) + +class EmptyStateNet(nn.Module): + def __init__(self, d_M, d_F): + super().__init__() + self.net=nn.Sequential(nn.Linear(d_M+d_F,2*d_F),nn.SiLU(),nn.LayerNorm(2*d_F), + nn.Linear(2*d_F,d_F)) + def forward(self, xq, fq): return self.net(torch.cat([xq,fq],-1)) + +class WriteGate(nn.Module): + def __init__(self, c): + super().__init__() + self.net=nn.Sequential(nn.Linear(c.d_LLM+1,c.d_LLM//4),nn.SiLU(),nn.Linear(c.d_LLM//4,1)) + def forward(self, h, surprise): + s=surprise.view(-1,1) if surprise.dim()>=1 else surprise.unsqueeze(0).unsqueeze(0) + if s.shape[0]!=h.shape[0]: s=s[:h.shape[0]] + return torch.sigmoid(self.net(torch.cat([h,s],-1)).squeeze(-1)) + +class RetentionScorer(nn.Module): + def __init__(self, c): + super().__init__() + self.net=nn.Sequential(nn.Linear(c.d_M+c.d_F+3,64),nn.SiLU(), + nn.Linear(64,64),nn.SiLU(),nn.Linear(64,1),nn.Sigmoid()) + def forward(self, base, fiber, surprise, dt, cnt): + return self.net(torch.cat([base,fiber, + surprise.unsqueeze(-1) if surprise.dim()==1 else surprise, + dt.unsqueeze(-1) if dt.dim()==1 else dt, + cnt.float().unsqueeze(-1) if cnt.dim()==1 else cnt.float()],-1)).squeeze(-1) + +class RetrievalReranker(nn.Module): + def __init__(self, d_M, d_F, clip=0.2): + super().__init__(); self.clip=clip + inp=2*d_M+2*d_F+1 + self.net=nn.Sequential(nn.Linear(inp,128),nn.SiLU(),nn.LayerNorm(128), + nn.Linear(128,64),nn.SiLU(),nn.LayerNorm(64),nn.Linear(64,1)) + nn.init.zeros_(self.net[-1].weight); nn.init.zeros_(self.net[-1].bias) + def forward(self, xq, fq, xc, fc, dir_sim): + B,C=xc.shape[:2] + xq_e=xq.unsqueeze(1).expand(-1,C,-1); fq_e=fq.unsqueeze(1).expand(-1,C,-1) + inp=torch.cat([xq_e,fq_e,xc,fc,dir_sim.unsqueeze(-1)],-1) + correction=self.net(inp).squeeze(-1) + return dir_sim + correction.clamp(-self.clip, self.clip) + +class ContentBypass(nn.Module): + def __init__(self, d_F, d_LLM, gate_bias=0.5): + super().__init__() + self.proj=nn.Sequential( + nn.Linear(d_F,2*d_LLM),nn.SiLU(),nn.LayerNorm(2*d_LLM), + nn.Linear(2*d_LLM,d_LLM),nn.LayerNorm(d_LLM)) + self.gate_net=nn.Sequential(nn.Linear(d_F+d_LLM,128),nn.SiLU(),nn.Linear(128,1)) + nn.init.constant_(self.gate_net[-1].bias,gate_bias) + nn.init.normal_(self.proj[3].weight,std=0.02); nn.init.zeros_(self.proj[3].bias) + self._last_gate=None + def forward(self, fiber_summary, qformer_context): + projected=self.proj(fiber_summary) + gate_in=torch.cat([fiber_summary,qformer_context],-1) + g=torch.sigmoid(self.gate_net(gate_in)); self._last_gate=g.detach() + return projected*g + +class PrefixSemanticProbe(nn.Module): + def __init__(self, d_LLM, L_mem, d_F): + super().__init__() + self.attn_pool=nn.Linear(d_LLM,1) + self.fiber_decode=nn.Sequential( + nn.Linear(d_LLM,2*d_F),nn.SiLU(),nn.LayerNorm(2*d_F),nn.Linear(2*d_F,d_F)) + def forward(self, prefix): + w=F.softmax(self.attn_pool(prefix).squeeze(-1),dim=1) + pooled=(w.unsqueeze(-1)*prefix).sum(1) + return self.fiber_decode(pooled) + +class PrefixAligner(nn.Module): + def __init__(self, d_LLM, init_scale=0.5): + super().__init__() + self.ln=nn.LayerNorm(d_LLM) + self.scale_logit=nn.Parameter(torch.tensor(init_scale)) + self.register_buffer('_target_std',torch.tensor(1.0)) + self._calibrated=False + def calibrate(self, wte_fp32): + with torch.no_grad(): + V = wte_fp32.shape[0] + si = min(5000, V) + idx = torch.randperm(V, device=wte_fp32.device)[:si] + sample = wte_fp32[idx] + self._target_std.fill_(float(sample.std().item())) + self._calibrated=True + def forward(self, prefix): + normed=self.ln(prefix) + scale=torch.sigmoid(self.scale_logit)*self._target_std + return normed*scale + +class ContentSemanticTailHead(nn.Module): + def __init__(self, d_F, d_LLM, n_slots, hidden=1024): + super().__init__() + self.n_slots = n_slots; self.d_LLM = d_LLM + if n_slots == 0: return + self.shared = nn.Sequential( + nn.Linear(d_F, hidden), nn.SiLU(), nn.LayerNorm(hidden), + nn.Linear(hidden, hidden), nn.SiLU(), nn.LayerNorm(hidden)) + self.slot_heads = nn.ModuleList([ + nn.Sequential(nn.Linear(hidden, d_LLM), nn.LayerNorm(d_LLM)) + for _ in range(n_slots)]) + for head in self.slot_heads: + nn.init.normal_(head[0].weight, std=0.02); nn.init.zeros_(head[0].bias) + def forward(self, fiber_summary): + if self.n_slots == 0: return None + h = self.shared(fiber_summary) + slots = [head(h) for head in self.slot_heads] + return torch.stack(slots, dim=1) + +class ContentTokenClassifier: + DEFAULT_STOPWORDS = frozenset({ + 'the','a','an','is','are','was','were','be','been','being', + 'have','has','had','having','do','does','did','doing', + 'will','would','could','should','may','might','can','shall', + 'and','but','or','nor','for','yet','so', + 'in','on','at','to','of','by','with','from','as','into','through', + 'during','before','after','above','below','between','under','over', + 'that','this','these','those','it','its', + 'he','she','they','we','you','me','him','her','them','us', + 'his','her','their','our','your','my','mine','yours', + 'not','no','if','then','than','when','where','what','which','who', + 'how','all','each','every','both','few','more','most','some','any', + 'also','just','about','very','really','only','even','still','already', + 'up','down','out','off','away','back','here','there','now', + 'too','much','many','such','own','other','another', + 'because','since','while','although','though','until','unless', + 'however','therefore','moreover','furthermore','nevertheless', + 'like','get','got','go','went','gone','come','came', + 'make','made','take','took','give','gave','see','saw','know','knew', + 'think','thought','say','said','tell','told','want','need', + 'use','used','find','found','put','keep','kept','let', + 'seem','become','became','leave','left','call','called', + 'try','tried','ask','asked','work','worked','well','way', + 'thing','things','something','anything','nothing','everything', + 'one','two','first','new','old','good','bad','big','small', + 'long','little','right','same','different','last','next', + 'part','being','going','using','getting','making','looking', + 'coming','taking','having','doing','saying','working','trying', + 'include','includes','including','included'}) + DEFAULT_FILLER_WORDS = frozenset({ + 'include','includes','including','included', + 'also','just','however','moreover','furthermore', + 'nevertheless','therefore','thus','hence','accordingly', + 'meanwhile','instead','rather','otherwise','additionally', + 'basically','essentially','actually','obviously','clearly', + 'simply','certainly','indeed','probably','perhaps', + 'apparently','presumably','supposedly','regardless', + 'nonetheless','conversely','alternatively','specifically', + 'generally','typically','usually','often','sometimes', + 'particularly','especially','notably', + 'various','several','many','multiple','different','diverse','varied', + 'certain','particular','specific','general','overall','whole','entire', + 'aspect','aspects','feature','features','element','elements', + 'factor','factors','component','components','quality','qualities', + 'example','examples','instance','instances','case','cases', + 'method','methods','approach','approaches','technique_generic', + 'process','processes','system','systems','part','parts', + 'kind','kinds','type','types','sort','sorts', + 'people','person','someone','anyone','everyone', + 'matter','matters','issue','issues','point','points', + 'number','numbers','amount','amounts','level','levels', + 'student','students','practice','practicing', + 'action','actions','role','roles','purpose','purposes', + 'nature','natures','character','characters','condition','conditions', + 'state','states','status','statuses','fact','facts', + 'substance','substances','material','materials','content','contents', + 'context','contexts','task','tasks','duty','duties', + 'operation','operations','performance','performances', + 'activity','activities','topic','topics','subject','subjects', + 'concept','concepts','idea','ideas','notion','notions', + 'result','results','outcome','outcomes','effect','effects', + 'area','areas','region','regions','range','ranges', + 'degree','degrees','extent','extents','period','periods', + 'moment','moments','detail','details','information', + 'piece','pieces','group','groups','set','sets', + 'form','forms','style','styles','mode','modes','version','versions', + 'manner','manners','fashion','fashions','attribute','attributes', + 'property','properties','trait','traits','characteristic','characteristics', + 'place','places','way','ways'}) + + def __init__(self, tokenizer, cfg=None, vocab_size=None, min_len=None, strict_min_len=None): + if cfg is None: cfg = Cfg() + self.cfg = cfg + _min_len = min_len if isinstance(min_len, int) else cfg.content_min_len + _strict_min_len = (strict_min_len if isinstance(strict_min_len, int) + else cfg.strict_starter_min_decoded_len) + self.STOPWORDS = (cfg.stopwords_override if cfg.stopwords_override is not None + else self.DEFAULT_STOPWORDS | cfg.stopwords_extra) + self.FILLER_WORDS = (cfg.filler_words_override if cfg.filler_words_override is not None + else self.DEFAULT_FILLER_WORDS | cfg.filler_words_extra) + if cfg.dedup_filler_from_stop: + self.FILLER_WORDS = self.FILLER_WORDS - self.STOPWORDS + self.content_ids = set(); self.function_ids = set() + self.punct_ids = set(); self.newline_ids = set() + self.filler_ids = set(); self.word_starter_ids = set() + self.content_starter_ids = set(); self.strict_content_starter_ids = set() + V = int(vocab_size) if vocab_size is not None else int(getattr(tokenizer, 'vocab_size', 50257)) + self._V = V + for i in range(V): + try: tok_text = tokenizer.decode([i]) + except Exception: + self.function_ids.add(i); continue + if not isinstance(tok_text, str): self.function_ids.add(i); continue + is_word_starter = len(tok_text) > 0 and tok_text[0] in (' ', '\t') + stripped = tok_text.strip().lower() + cleaned = ''.join(c for c in stripped if c.isalpha()) + if is_word_starter: self.word_starter_ids.add(i) + if '\n' in tok_text: + self.newline_ids.add(i); self.function_ids.add(i) + elif stripped == '' or all(not c.isalnum() for c in stripped): + self.punct_ids.add(i); self.function_ids.add(i) + elif len(cleaned) >= _min_len and cleaned not in self.STOPWORDS: + self.content_ids.add(i) + if is_word_starter: + self.content_starter_ids.add(i) + if (stripped == cleaned and len(stripped) >= _strict_min_len + and stripped not in self.STOPWORDS + and stripped not in self.FILLER_WORDS): + self.strict_content_starter_ids.add(i) + else: self.function_ids.add(i) + if cleaned in self.FILLER_WORDS: self.filler_ids.add(i) + self._content_tensor = None; self._content_starter_tensor = None + self._strict_content_starter_tensor = None; self._filler_tensor = None + + def _mask_size(self): return int(self._V) + def content_mask(self, device): + if self._content_tensor is None or self._content_tensor.device != device: + V = self._mask_size(); m = torch.zeros(V, device=device) + for i in self.content_ids: + if i < V: m[i] = 1.0 + self._content_tensor = m + return self._content_tensor + def content_starter_mask(self, device): + if self._content_starter_tensor is None or self._content_starter_tensor.device != device: + V = self._mask_size(); m = torch.zeros(V, device=device) + for i in self.content_starter_ids: + if i < V: m[i] = 1.0 + self._content_starter_tensor = m + return self._content_starter_tensor + def strict_content_starter_mask(self, device): + if (self._strict_content_starter_tensor is None + or self._strict_content_starter_tensor.device != device): + V = self._mask_size(); m = torch.zeros(V, device=device) + for i in self.strict_content_starter_ids: + if i < V: m[i] = 1.0 + self._strict_content_starter_tensor = m + return self._strict_content_starter_tensor + def filler_mask(self, device): + if self._filler_tensor is None or self._filler_tensor.device != device: + V = self._mask_size(); m = torch.zeros(V, device=device) + for i in self.filler_ids: + if i < V: m[i] = 1.0 + self._filler_tensor = m + return self._filler_tensor + def get_content_ids_from_tokens(self, token_ids): + return [t for t in token_ids if t in self.content_ids] + +class MemoryVocabProjector(nn.Module): + def __init__(self, d_F, d_LLM): + super().__init__() + self.proj = nn.Sequential( + nn.Linear(d_F, 4*d_LLM), nn.SiLU(), nn.LayerNorm(4*d_LLM), + nn.Linear(4*d_LLM, 2*d_LLM), nn.SiLU(), nn.LayerNorm(2*d_LLM), + nn.Linear(2*d_LLM, d_LLM)) + nn.init.zeros_(self.proj[-1].weight); nn.init.zeros_(self.proj[-1].bias) + def forward(self, fiber_summary, wte_weight): + mem_emb = self.proj(fiber_summary) + mem_n = F.normalize(mem_emb, dim=-1, eps=1e-8) + wte_n = F.normalize(wte_weight, dim=-1, eps=1e-8) + return mem_n @ wte_n.T + +@dataclass +class MemEntry: + mid: int; base: torch.Tensor; fiber: torch.Tensor; dirn: torch.Tensor + surprise: float; ts: float; last: float; cnt: int = 0; version: int = 0 + source_text: str = "" + content_token_ids: List[int] = field(default_factory=list) + semantic_emb: Optional[torch.Tensor] = None + expanded_content_ids: List[int] = field(default_factory=list) + +class _Node: + __slots__=('leaf','ids','children','centers','depth') + def __init__(self,d=0): + self.depth=d; self.leaf=True; self.ids=[]; self.children=[]; self.centers=None + def count(self): + return len(self.ids) if self.leaf else sum(c.count() for c in self.children) + +class DirectionTree: + def __init__(self, c): + self.c=c; self.root=_Node(); self.store={}; self.nid=0 + def insert(self, m): + self.store[m.mid]=m; self._ins(self.root,m) + def _ins(self, nd, m): + if nd.leaf: + nd.ids.append(m.mid) + if len(nd.ids)>self.c.tree_max_leaf: self._split(nd) + else: + best=self._best(nd,m.dirn); self._ins(nd.children[best],m); self._update_centers(nd) + def update(self, mid, new_base=None, new_fiber=None, new_dirn=None): + if mid not in self.store: return + m=self.store[mid]; dc=False + if new_base is not None: m.base=new_base.detach().clone() + if new_fiber is not None: m.fiber=new_fiber.detach().clone() + if new_dirn is not None: dc=True; m.dirn=new_dirn.detach().clone() + m.version+=1 + if dc: self._rm(self.root,mid); self._ins(self.root,m); self._rebalance(self.root) + def _split(self, nd): + ids=nd.ids + if len(ids)<2: return + K=min(self.c.tree_K,len(ids)) + if K<2: return + dirs=torch.stack([self.store[i].dirn for i in ids]) + centered=dirs-dirs.mean(0) + try: _,_,Vh=torch.linalg.svd(centered,full_matrices=False) + except: return + n_comp=min(K,dirs.shape[1]); proj=centered@Vh[:n_comp].T + asgn=self._farthest_kmeans(proj,K) + children=[] + for k in range(K): + ch=_Node(nd.depth+1); ch.ids=[ids[i] for i in range(len(ids)) if asgn[i]==k] + if ch.ids: children.append(ch) + if len(children)<=1: return + nd.leaf=False; nd.children=children; nd.ids=[]; self._update_centers(nd) + for ch in nd.children: + if ch.leaf and len(ch.ids)>self.c.tree_max_leaf: self._split(ch) + @staticmethod + def _farthest_kmeans(data, K, max_iter=50): + N=data.shape[0]; K=min(K,N) + if K<=0: return torch.zeros(N,dtype=torch.long,device=data.device) + ctrs=[data[0].clone()] + for _ in range(K-1): + d2=torch.cdist(data,torch.stack(ctrs)).min(1)[0].pow(2) + ctrs.append(data[d2.argmax()].clone()) + ctrs=torch.stack(ctrs); asgn=torch.zeros(N,dtype=torch.long,device=data.device) + for _ in range(max_iter): + dists=torch.cdist(data,ctrs); new=dists.argmin(1) + if (new==asgn).all(): break + asgn=new + for k in range(K): + mk=asgn==k + if mk.any(): ctrs[k]=data[mk].mean(0) + else: + far=dists.min(1)[0].argmax(); ctrs[k]=data[far].clone(); asgn[far]=k + return asgn + def _best(self, nd, d): + if nd.centers is None or len(nd.children)==0: return 0 + return (nd.centers@d).argmax().item() + def retrieve(self, qdir, bw=3): + beams=[(self.root,0.)]; results={} + while beams: + nb=[] + for nd,sc in beams: + if nd.leaf: + for mid in nd.ids: + if mid in self.store: + s=(qdir@self.store[mid].dirn).item()+sc + if mid not in results or s>results[mid]: results[mid]=s + elif nd.centers is not None: + sims=nd.centers@qdir; tk=min(bw,len(nd.children)); _,idxs=sims.topk(tk) + for i in idxs: nb.append((nd.children[i.item()],sc+sims[i.item()].item())) + else: + for ch in nd.children: nb.append((ch,sc)) + nb.sort(key=lambda x:-x[1]); beams=nb[:bw] + return sorted(results.items(),key=lambda x:-x[1]) + def remove(self, mid): + if mid not in self.store: return + del self.store[mid]; self._rm(self.root,mid); self._rebalance(self.root) + def _rm(self, nd, mid): + if nd.leaf: + if mid in nd.ids: nd.ids.remove(mid); return True + return False + return any(self._rm(c,mid) for c in nd.children) + def _rebalance(self, nd): + if nd.leaf: return + for c in nd.children: self._rebalance(c) + nd.children=[c for c in nd.children if c.count()>0] + if not nd.children: nd.leaf=True; nd.ids=[]; nd.centers=None + elif len(nd.children)==1: + ch=nd.children[0]; nd.leaf=ch.leaf; nd.ids=ch.ids; nd.children=ch.children; nd.centers=ch.centers + else: self._update_centers(nd) + def _update_centers(self, nd): + cs=[] + for c in nd.children: + ids=self._collect(c); dirs=[self.store[i].dirn for i in ids if i in self.store] + if not dirs: continue + cs.append(F.normalize(torch.stack(dirs).mean(0),dim=0)) + nd.centers=torch.stack(cs) if cs else None + def _collect(self, nd): + if nd.leaf: return list(nd.ids) + return [i for c in nd.children for i in self._collect(c)] + def rebuild(self): + ms=list(self.store.values()); self.root=_Node() + for m in ms: self._ins(self.root,m) + def verify_consistency(self): + errs=[]; ti=set(self._collect(self.root)); si=set(self.store.keys()) + if ti!=si: errs.append(f"tree≠store: tree_only={ti-si}, store_only={si-ti}") + if self.root.count()!=len(self.store): errs.append(f"count mismatch") + return errs + + def max_depth(self) -> int: + def _d(nd): + if nd.leaf: return nd.depth + if not nd.children: return nd.depth + return max(_d(c) for c in nd.children) + return _d(self.root) + + def leaf_size_violations(self) -> List[Tuple[int, int]]: + viols: List[Tuple[int, int]] = [] + def _check(nd): + if nd.leaf: + if len(nd.ids) > self.c.tree_max_leaf: + viols.append((nd.depth, len(nd.ids))) + else: + for c in nd.children: _check(c) + _check(self.root) + return viols + +class FiberAttn(nn.Module): + def __init__(self, c): + super().__init__() + self.nh=c.n_heads_fiber; self.hd=c.d_F//c.n_heads_fiber + self.Wq=nn.Linear(c.d_F,c.d_F,bias=False); self.Wk=nn.Linear(c.d_F,c.d_F,bias=False) + self.Wv=nn.Linear(c.d_F,c.d_F,bias=False); self.Wo=nn.Linear(c.d_F,c.d_F,bias=False) + self.n1=nn.LayerNorm(c.d_F) + self.ff=nn.Sequential(nn.Linear(c.d_F,2*c.d_F),nn.GELU(),nn.Linear(2*c.d_F,c.d_F)) + self.n2=nn.LayerNorm(c.d_F) + def forward(self, qf, mf, mem_mask=None, dir_bias=None): + B,C,d=mf.shape; nh=self.nh; hd=self.hd; S=1+C + seq=torch.cat([qf.unsqueeze(1),mf],1) + Q=self.Wq(seq).reshape(B,S,nh,hd).permute(0,2,1,3) + K=self.Wk(seq).reshape(B,S,nh,hd).permute(0,2,1,3) + V=self.Wv(seq).reshape(B,S,nh,hd).permute(0,2,1,3) + a=(Q@K.transpose(-2,-1))/math.sqrt(hd) + if dir_bias is not None: + db=dir_bias.unsqueeze(1).unsqueeze(2) + pad=torch.zeros(B,1,1,1,**_dev(a)); a=a+torch.cat([pad,db],-1) + if mem_mask is not None: + qm=torch.ones(B,1,**_dev(mem_mask)); full=torch.cat([qm,mem_mask],1) + a=a.masked_fill(full.unsqueeze(1).unsqueeze(2)==0,-1e9) + a=F.softmax(a,-1); out=(a@V).permute(0,2,1,3).reshape(B,S,d) + out=self.n1(seq+self.Wo(out)); out=self.n2(out+self.ff(out)) + return out[:,1:] + +class QFormerLayer(nn.Module): + def __init__(self, c): + super().__init__(); d=c.d_LLM; nh=c.bridge_heads + self.sa=nn.MultiheadAttention(d,nh,batch_first=True) + self.ca=nn.MultiheadAttention(d,nh,batch_first=True) + self.ff=nn.Sequential(nn.Linear(d,4*d),nn.GELU(),nn.Linear(4*d,d)) + self.n1=nn.LayerNorm(d); self.n2=nn.LayerNorm(d); self.n3=nn.LayerNorm(d) + def forward(self, q, k, v, kv_mask=None): + h=self.n1(q); q=q+self.sa(h,h,h)[0]; h=self.n2(q) + kpm=None + if kv_mask is not None: + kpm=(kv_mask==0); all_m=kpm.all(dim=-1) + if all_m.any(): kpm=kpm.clone(); kpm[all_m]=False + q=q+self.ca(h,k,v,key_padding_mask=kpm)[0] + return q+self.ff(self.n3(q)) + +class QFormerProj(nn.Module): + def __init__(self, c): + super().__init__() + self.q=nn.Parameter(torch.randn(c.L_mem,c.d_LLM)*0.02) + self.fkv=nn.Linear(c.d_F,c.d_LLM*2) + self.layers=nn.ModuleList([QFormerLayer(c) for _ in range(c.bridge_layers)]) + self.norm=nn.LayerNorm(c.d_LLM) + def forward(self, fibers, mem_mask=None): + B=fibers.shape[0]; kv=self.fkv(fibers); k,v=kv.chunk(2,-1) + q=self.q.unsqueeze(0).expand(B,-1,-1) + for l in self.layers: q=l(q,k,v,kv_mask=mem_mask) + return self.norm(q) + +class AdaptiveLayerPool(nn.Module): + def __init__(self, n, d): + super().__init__(); self.w=nn.Parameter(torch.linspace(-2,2,n)) + def forward(self, hs): + w=F.softmax(self.w,0); return sum(w[i]*h for i,h in enumerate(hs)) + def weight_dist(self): return F.softmax(self.w.detach(),0) + +class StateExtractor(nn.Module): + def __init__(self, c): + super().__init__(); pos_dim=5 + self.sc=nn.Sequential(nn.Linear(c.d_LLM+pos_dim,c.d_LLM//4),nn.Tanh(), + nn.Linear(c.d_LLM//4,1)) + self.tb=nn.Linear(c.d_LLM,c.d_M); self.tf=nn.Linear(c.d_LLM,c.d_F) + def _pos_feat(self, T, ref): + pos=torch.linspace(0,1,T,**_dev(ref)) + return torch.stack([pos,torch.sin(pos*math.pi),torch.cos(pos*math.pi), + torch.sin(2*pos*math.pi),torch.cos(2*pos*math.pi)],-1) + def forward(self, h, mask=None): + B,T,_=h.shape; pf=self._pos_feat(T,h).unsqueeze(0).expand(B,-1,-1) + s=self.sc(torch.cat([h,pf],-1)).squeeze(-1) + if mask is not None and mask.shape[1]==T: + s=s.masked_fill(mask==0,-1e9) + w=F.softmax(s,-1); p=(w.unsqueeze(-1)*h).sum(1) + return self.tb(p), self.tf(p) + +class EmbBridge(nn.Module): + def __init__(self, c): + super().__init__(); self.c=c + self.proj=QFormerProj(c); self.ext=StateExtractor(c) + self.pe=nn.Parameter(torch.randn(c.L_mem,c.d_LLM)*0.02) + self.bypass=ContentBypass(c.d_F,c.d_LLM,gate_bias=c.bypass_init_gate_bias) + self.aligner=PrefixAligner(c.d_LLM,c.prefix_init_scale) + self.tail_head = ContentSemanticTailHead( + c.d_F, c.d_LLM, + n_slots=c.content_tail_slots if c.use_content_semantic_tail else 0, + hidden=c.tail_head_hidden) + self._last_inject_diag={} + self._last_fiber_summary=None + self._last_tail_slots=None + + def _build_body_prefix(self, fibers, mem_mask, fiber_summary): + qf_out = self.proj(fibers, mem_mask) + self.pe.unsqueeze(0) + bp_out = None; gate_val = None + if fiber_summary is not None: + qf_context = qf_out.mean(1) + bp_out = self.bypass(fiber_summary, qf_context) + gate_val = self.bypass._last_gate + qf_out = qf_out + bp_out.unsqueeze(1) + qf_out = self.aligner(qf_out) + return qf_out, bp_out, gate_val + + def _apply_filler_projection_and_clamp(self, qf_out, filler_centroid): + L = qf_out.shape[1]; filler_dir_used = False + if self.c.use_filler_direction_projection and filler_centroid is not None: + n_proj = min(self.c.filler_projection_last_slots, L) + fd = filler_centroid.view(1, 1, -1) + mask_slot = torch.zeros(L, device=qf_out.device) + mask_slot[L - n_proj:] = 1.0 + mask_slot = mask_slot.view(1, -1, 1) + comp = (qf_out * fd).sum(-1, keepdim=True) + qf_out = qf_out - comp * fd * mask_slot + filler_dir_used = True + if self.c.use_prefix_norm_clamp: + target_std = self.aligner._target_std.item() + target_norm = target_std * math.sqrt(self.c.d_LLM) + max_allowed = target_norm * self.c.prefix_norm_clamp_ratio + slot_norms = qf_out.norm(dim=-1, keepdim=True).clamp(min=1e-8) + scale = torch.clamp(max_allowed / slot_norms, max=1.0) + qf_out = qf_out * scale + return qf_out, filler_dir_used + + def inject(self, fibers, mem_mask=None, fiber_summary=None, filler_centroid=None): + qf_out, bp_out, gate_val = self._build_body_prefix(fibers, mem_mask, fiber_summary) + tail_slots_used = 0 + if (self.c.use_content_semantic_tail and self.c.content_tail_slots > 0 + and fiber_summary is not None): + tail = self.tail_head(fiber_summary); tail = self.aligner(tail) + n = self.c.content_tail_slots + qf_out = torch.cat([qf_out[:, :-n, :], tail], dim=1) + tail_slots_used = n + self._last_tail_slots = tail.detach() + else: + self._last_tail_slots = None + qf_out, filler_dir_used = self._apply_filler_projection_and_clamp(qf_out, filler_centroid) + self._last_fiber_summary = (fiber_summary.detach() + if fiber_summary is not None else None) + self._last_inject_diag = { + 'bypass_gate': gate_val.mean().item() if gate_val is not None else None, + 'qf_norm': qf_out.norm().item(), + 'bypass_norm': bp_out.norm().item() if bp_out is not None else 0.0, + 'aligner_scale': (torch.sigmoid(self.aligner.scale_logit).item() + * self.aligner._target_std.item()), + 'last_slot_norm_per_b': qf_out[:, -1].norm(dim=-1).mean().item(), + 'tail_slots_used': tail_slots_used, + 'filler_dir_projected': filler_dir_used} + return qf_out + + def build_neutral_prefix(self, B, device): + qf_out = self.pe.unsqueeze(0).expand(B, -1, -1).contiguous() + qf_out = self.aligner(qf_out) + if self.c.use_prefix_norm_clamp: + target_std = self.aligner._target_std.item() + target_norm = target_std * math.sqrt(self.c.d_LLM) + max_allowed = target_norm * self.c.prefix_norm_clamp_ratio + slot_norms = qf_out.norm(dim=-1, keepdim=True).clamp(min=1e-8) + scale = torch.clamp(max_allowed / slot_norms, max=1.0) + qf_out = qf_out * scale + return qf_out + +class LossWarmup: + def __init__(self, schedules): self.schedules=schedules; self.step_count=0 + def weight(self, name): + ws=self.schedules.get(name,0) + if ws<=0: return 1.0 + return min(1.0, self.step_count/max(ws,1)) + def advance(self): self.step_count+=1 + +class GradientMonitor: + def __init__(self): self._groups={} + def register(self, name, mod): self._groups[name]=mod + def snapshot(self): + norms={} + for name,mod in self._groups.items(): + total=0.0; cnt=0 + for p in mod.parameters(): + if p.grad is not None: total+=p.grad.norm().item()**2; cnt+=1 + norms[name]=math.sqrt(total) if cnt>0 else 0.0 + return norms + +class DegenerationGuard: + def __init__(self, tok, cfg, content_classifier=None): + self.tok=tok; self.cfg=cfg; self.cc=content_classifier + def process(self, logits, generated_ids, step): + punct_ids = self.cc.punct_ids if self.cc else set() + newline_ids = self.cc.newline_ids if self.cc else set() + V = logits.shape[-1] + if step < self.cfg.early_content_steps: + pen_p = self.cfg.degen_early_punct_penalty + pen_n = self.cfg.degen_early_newline_penalty + for pid in punct_ids: + if pid < V: logits[0, pid] -= pen_p + for nid in newline_ids: + if nid < V: logits[0, nid] -= pen_n + if step < self.cfg.degen_min_tokens and self.tok.eos_token_id is not None: + if self.tok.eos_token_id < V: + logits[0, self.tok.eos_token_id] = -float('inf') + seen = set(generated_ids[-30:]) if generated_ids else set() + for tid in seen: + if tid < V: + if logits[0, tid] > 0: logits[0, tid] /= self.cfg.degen_repeat_penalty + else: logits[0, tid] *= self.cfg.degen_repeat_penalty + mc = self.cfg.degen_max_consec_punct + if len(generated_ids) >= mc: + recent = generated_ids[-mc:] + if all(t in punct_ids for t in recent): + for pid in punct_ids: + if pid < V: logits[0, pid] -= 10.0 + return logits + +@dataclass +class RetrievalDiag: + was_flat_scan: bool = False + recall_count: int = 0 + reranker_delta_mean: float = 0.0 + fiber_summary_norm: float = 0.0 + top_reranker_score: float = 0.0 + top_dir_sim: float = 0.0; top_sem_sim: float = 0.0 + top_forward_maxsim: float = 0.0; top_backward_maxsim: float = 0.0 + top_bidi_min: float = 0.0; top_gate_affinity: float = 0.0; gate_threshold: float = 0.0 + n_gate_pass: int = 0; n_candidates_initial: int = 0 + n_after_strict_overlap_gate: int = 0; n_after_upstream_semantic_gate: int = 0 + n_after_hard_filter: int = 0; n_after_score_filter: int = 0 + n_after_coherence_filter: int = 0; n_after_bidi_gap_filter: int = 0 + n_after_mean_center: int = 0 + mean_center_applied: bool = False + mean_center_dropped_ids: List[int] = field(default_factory=list) + mean_center_raw_scores: Dict[int, float] = field(default_factory=dict) + mean_center_final_scores: Dict[int, float] = field(default_factory=dict) + hungarian_used: bool = False + batch_mem_weights: List[List[Tuple[int, float]]] = field(default_factory=list) + per_memory_forward_maxsim: Dict[int, float] = field(default_factory=dict) + per_memory_bidi_min: Dict[int, float] = field(default_factory=dict) + per_memory_sem_sim: Dict[int, float] = field(default_factory=dict) + per_memory_gate_affinity: Dict[int, float] = field(default_factory=dict) + per_memory_strict_overlap: Dict[int, int] = field(default_factory=dict) + dominant_per_batch: List[Optional[int]] = field(default_factory=list) + dominant_memory_id: Optional[int] = None + non_dominant_per_batch: List[List[int]] = field(default_factory=list) + non_dominant_weights_per_batch: List[Dict[int, float]] = field(default_factory=list) + idf_applied: bool = False; centroid_applied: bool = False + top_centroid_cosine: float = 0.0 + per_memory_centroid_cosine: Dict[int, float] = field(default_factory=dict) + upstream_semantic_gate_applied: bool = False + upstream_gate_dropped_ids: List[int] = field(default_factory=list) + strict_overlap_gate_applied: bool = False + strict_overlap_dropped_ids: List[int] = field(default_factory=list) + +class AMM(nn.Module): + def __init__(self, c): + super().__init__(); self.c=c + self.metric=RiemannianMetric(c.d_M) + self.geo=GeodesicSolver(self.metric,c) + self.conn=FiberConnection(c.d_M,c.d_F,self.metric,grad_coupling=True) + self.trans=FiberTransporter(self.conn,c) + self.ctx=CtxEncoder(c); self.fib=FibEncoder(c) + self.dir_pred=DirectionPredictor(c.d_M,c.d_F) + self.write_gate=WriteGate(c); self.retention=RetentionScorer(c) + self.attn=FiberAttn(c); self.empty_state=EmptyStateNet(c.d_M,c.d_F) + self.contrast_proj_f=nn.Linear(c.d_F,c.d_M,bias=False) + self.contrast_proj_x=nn.Linear(c.d_M,c.d_M,bias=False) + nn.init.eye_(self.contrast_proj_x.weight) + self.reranker=RetrievalReranker(c.d_M,c.d_F,clip=c.reranker_clip) + self.tree=DirectionTree(c); self.time=0. + self.wte_normed = None + + def surprise_proxy(self, logits, tgt): + nll=-F.log_softmax(logits,-1).gather(2,tgt.unsqueeze(-1)).squeeze(-1) + T=nll.shape[1] + if T==0: return logits.new_zeros(logits.shape[0]) + w=torch.linspace(0.5,1.5,T,**_dev(nll)); w=w/w.sum()*T + return (nll*w.unsqueeze(0)).mean(-1) + + def _compute_dirn(self, base, fiber): + with torch.no_grad(): + return self.dir_pred(base.unsqueeze(0),fiber.unsqueeze(0)).squeeze(0) + + def _get_mem_scoring_ids(self, mem): + if self.c.retrieval_use_expanded_ids and mem.expanded_content_ids: + return mem.expanded_content_ids + return mem.content_token_ids + + def _compute_corpus_idf(self, content_classifier): + s = self.c.tfidf_smoothing + N = len(self.tree.store) + if N == 0: return {} + df = {} + for mem in self.tree.store.values(): + label_set = (set(t for t in mem.content_token_ids + if t in content_classifier.content_starter_ids) + if content_classifier is not None else set(mem.content_token_ids)) + for t in label_set: df[t] = df.get(t, 0) + 1 + return {t: math.log((N + s) / (d + s)) + 1.0 for t, d in df.items()} + + @staticmethod + def _compute_idf_weighted_centroid(token_ids, wte_normed, corpus_idf, idf_floor=0.1): + if not token_ids or wte_normed is None: return None + V = wte_normed.shape[0] + valid = [t for t in token_ids if t < V] + if not valid: return None + if corpus_idf is not None and len(corpus_idf) > 0: + weights = torch.tensor( + [max(corpus_idf.get(t, idf_floor), idf_floor) for t in valid], + device=wte_normed.device, dtype=wte_normed.dtype) + else: + weights = torch.ones(len(valid), device=wte_normed.device, dtype=wte_normed.dtype) + vecs = wte_normed[valid] + centroid = (vecs * weights.unsqueeze(1)).sum(0) / weights.sum().clamp(min=1e-8) + return F.normalize(centroid, dim=-1, eps=1e-8) + + def _compute_forward_hungarian(self, query_ids, mem_ids, wte_normed, query_idf=None, idf_floor=0.1): + if not query_ids or not mem_ids: return 0.0 + V = wte_normed.shape[0] + q_valid = [q for q in query_ids if q < V] + m_valid = [m for m in mem_ids if m < V] + if not q_valid or not m_valid: return 0.0 + n_q, n_m = len(q_valid), len(m_valid) + q_vecs = wte_normed[q_valid]; m_vecs = wte_normed[m_valid] + sim = q_vecs @ m_vecs.T + if max(n_q, n_m) > self.c.hungarian_max_n: + max_per_q = sim.max(dim=1).values + if query_idf is not None: + w = torch.tensor( + [max(query_idf.get(q, idf_floor), idf_floor) for q in q_valid], + device=wte_normed.device, dtype=sim.dtype) + return ((max_per_q * w).sum() / w.sum().clamp(min=1e-8)).item() + return max_per_q.mean().item() + pairs, _ = hungarian_max_assignment(sim) + if pairs.numel() == 0: return 0.0 + matched_sims = sim[pairs[:, 0], pairs[:, 1]] + if query_idf is not None: + q_ids_for_pairs = [q_valid[int(r.item())] for r in pairs[:, 0]] + w = torch.tensor( + [max(query_idf.get(q, idf_floor), idf_floor) for q in q_ids_for_pairs], + device=wte_normed.device, dtype=matched_sims.dtype) + return ((matched_sims * w).sum() / w.sum().clamp(min=1e-8)).item() + return matched_sims.mean().item() + + @staticmethod + def _compute_forward_maxsim(query_ids, mem_ids, wte_normed, query_idf=None, idf_floor=0.1): + if not query_ids or not mem_ids: return 0.0 + V = wte_normed.shape[0] + q_valid = [q for q in query_ids if q < V] + m_valid = [m for m in mem_ids if m < V] + if not q_valid or not m_valid: return 0.0 + q_vecs = wte_normed[q_valid]; m_vecs = wte_normed[m_valid] + sim = q_vecs @ m_vecs.T + max_per_q = sim.max(dim=1).values + if query_idf is not None: + weights = torch.tensor( + [max(query_idf.get(q, idf_floor), idf_floor) for q in q_valid], + device=wte_normed.device, dtype=sim.dtype) + total = weights.sum().clamp(min=1e-8) + return ((max_per_q * weights).sum() / total).item() + return max_per_q.mean().item() + + @staticmethod + def _compute_backward_maxsim(query_ids, mem_ids, wte_normed, query_idf=None, idf_floor=0.1): + if not query_ids or not mem_ids: return 0.0 + V = wte_normed.shape[0] + q_valid = [q for q in query_ids if q < V] + m_valid = [m for m in mem_ids if m < V] + if not q_valid or not m_valid: return 0.0 + q_vecs = wte_normed[q_valid]; m_vecs = wte_normed[m_valid] + sim = q_vecs @ m_vecs.T + max_per_m_vals, max_per_m_idx = sim.max(dim=0) + if query_idf is not None: + q_weights = torch.tensor( + [max(query_idf.get(q, idf_floor), idf_floor) for q in q_valid], + device=wte_normed.device, dtype=sim.dtype) + matched_weights = q_weights[max_per_m_idx] + total = matched_weights.sum().clamp(min=1e-8) + return ((max_per_m_vals * matched_weights).sum() / total).item() + return max_per_m_vals.mean().item() + + def _compute_bidi_min(self, q_ids, m_ids, wte_normed, query_idf, idf_floor): + fwd = (self._compute_forward_hungarian(q_ids, m_ids, wte_normed, query_idf, idf_floor) + if self.c.use_hungarian_fwd + else self._compute_forward_maxsim(q_ids, m_ids, wte_normed, query_idf, idf_floor)) + bwd = self._compute_backward_maxsim(q_ids, m_ids, wte_normed, query_idf, idf_floor) + return fwd, bwd, min(fwd, bwd) + + @staticmethod + def _count_strict_overlap_matches(q_strict_ids, m_strict_ids, wte_normed, sim_threshold): + if not q_strict_ids or not m_strict_ids or wte_normed is None: return 0 + V = wte_normed.shape[0] + q_valid = [t for t in q_strict_ids if t < V] + m_valid = [t for t in m_strict_ids if t < V] + if not q_valid or not m_valid: return 0 + dev = wte_normed.device + q_vecs = wte_normed[torch.tensor(q_valid, device=dev)] + m_vecs = wte_normed[torch.tensor(m_valid, device=dev)] + sim = q_vecs @ m_vecs.T + has_match = (sim >= sim_threshold).any(dim=1) + return int(has_match.sum().item()) + + def _check_consolidation_compatible(self, existing_content_ids, new_content_ids): + if not existing_content_ids or not new_content_ids: return True + if self.wte_normed is None: return True + _, _, m = self._compute_bidi_min(existing_content_ids, new_content_ids, + self.wte_normed, None, self.c.idf_floor) + return m >= self.c.consol_maxsim_min + + def store_mem(self, h, surp, training_mode=False, source_text="", + content_token_ids=None, content_semantic_emb=None, expanded_content_ids=None): + dev=h.device; h2=h.unsqueeze(0) + x=self.ctx(h2).squeeze(0).detach() + s=surp if isinstance(surp,torch.Tensor) else torch.tensor(surp,**_dev(h)) + sv=s.view(1) if s.dim()<=1 else s + f=self.fib(h2,x.unsqueeze(0),sv).squeeze(0).detach() + d=self._compute_dirn(x,f) + sem_emb=content_semantic_emb if content_semantic_emb is not None else h.detach().clone() + ct_ids=content_token_ids or []; exp_ids=expanded_content_ids or [] + if self.tree.store: + scored=self.tree.retrieve(d.detach(),bw=1)[:5] + for mid,_ in scored: + if mid in self.tree.store: + ex=self.tree.store[mid] + dist=self.metric.midpoint_approx_distance( + x.unsqueeze(0),ex.base.unsqueeze(0).to(dev)).item() + if dist= self.c.strict_overlap_min_matches + n_pass = int(pass_mask.sum().item()) + if n_pass < self.c.strict_overlap_min_keep: + keep_n = max(self.c.strict_overlap_min_keep, 1) + _, top_keep = overlap_counts.topk(min(keep_n, len(mems))) + pass_mask = torch.zeros(len(mems), dtype=torch.bool, device=dev) + pass_mask[top_keep] = True + dropped_local = (~pass_mask).nonzero(as_tuple=True)[0].tolist() + diag.strict_overlap_dropped_ids = [mems[i].mid for i in dropped_local] + diag.strict_overlap_gate_applied = True + keep_local = pass_mask.nonzero(as_tuple=True)[0] + if keep_local.numel() < len(mems): + mems = [mems[i] for i in keep_local.tolist()] + diag.n_after_strict_overlap_gate = len(mems) + C_init = len(mems) + if C_init == 0: + empty=self.empty_state(xq[b:b+1],fq[b:b+1]) + all_results.append(empty.squeeze(0).unsqueeze(0)) + all_masks.append(torch.ones(1,**_dev(xq))) + all_biases.append(torch.zeros(1,**_dev(xq))) + all_summaries.append(empty.squeeze(0)) + all_batch_mw.append([]); all_dominant.append(None) + all_non_dominant.append([]); all_non_dom_weights.append({}) + continue + sb_all=torch.stack([m.base.to(dev) for m in mems]) + sf_all=torch.stack([m.fiber.to(dev) for m in mems]) + md_all=torch.stack([m.dirn.to(dev) for m in mems]) + sem_sim_all=torch.zeros(C_init, device=dev) + if query_semantic_emb is not None: + for mi, mem in enumerate(mems): + if mem.semantic_emb is not None: + sem_sim_all[mi] = F.cosine_similarity( + query_semantic_emb[b:b+1], + mem.semantic_emb.unsqueeze(0).to(dev),dim=-1).squeeze() + forward_all=torch.zeros(C_init, device=dev) + backward_all=torch.zeros(C_init, device=dev) + bidi_min_all=torch.zeros(C_init, device=dev) + if q_content_ids and wn is not None: + for mi, mem in enumerate(mems): + scoring_ids = self._get_mem_scoring_ids(mem) + fwd, bwd, bmin = self._compute_bidi_min( + q_content_ids, scoring_ids, wn, corpus_idf, idf_floor) + forward_all[mi] = fwd; backward_all[mi] = bwd; bidi_min_all[mi] = bmin + if self.c.use_upstream_semantic_gate and q_content_ids and wn is not None: + fwd_pass = forward_all >= self.c.upstream_gate_fwd_idf_floor + sem_pass = sem_sim_all >= self.c.upstream_gate_sem_floor + pass_mask = (fwd_pass & sem_pass) if self.c.upstream_gate_require_both else (fwd_pass | sem_pass) + n_pass = int(pass_mask.sum().item()) + if n_pass < self.c.upstream_gate_min_keep: + keep_n = max(self.c.upstream_gate_min_keep, 1) + top_keep = forward_all.topk(min(keep_n, C_init)).indices + pass_mask = torch.zeros(C_init, dtype=torch.bool, device=dev) + pass_mask[top_keep] = True + dropped_local = (~pass_mask).nonzero(as_tuple=True)[0].tolist() + if dropped_local: + diag.upstream_gate_dropped_ids = [mems[i].mid for i in dropped_local] + diag.upstream_semantic_gate_applied = True + keep_local = pass_mask.nonzero(as_tuple=True)[0] + if keep_local.numel() < C_init: + mems = [mems[i] for i in keep_local.tolist()] + sb_all = sb_all[keep_local]; sf_all = sf_all[keep_local] + md_all = md_all[keep_local]; sem_sim_all = sem_sim_all[keep_local] + forward_all = forward_all[keep_local] + backward_all = backward_all[keep_local] + bidi_min_all = bidi_min_all[keep_local] + C_init = len(mems) + diag.n_after_upstream_semantic_gate = C_init + sb = sb_all; sf = sf_all + sem_sim_t = sem_sim_all; forward_t = forward_all; bidi_min_t = bidi_min_all + raw_dir_sim = torch.einsum('d,cd->c', qdir[b], md_all) + diag.top_dir_sim = raw_dir_sim.max().item() if C_init > 0 else 0.0 + diag.top_sem_sim = sem_sim_t.max().item() if C_init > 0 else 0.0 + diag.top_forward_maxsim = forward_t.max().item() if C_init > 0 else 0.0 + diag.top_backward_maxsim = backward_all.max().item() if C_init > 0 else 0.0 + diag.top_bidi_min = bidi_min_t.max().item() if C_init > 0 else 0.0 + centroid_scores = torch.zeros(C_init, device=dev) + if self.c.use_idf_centroid and q_content_ids and wn is not None: + q_centroid = self._compute_idf_weighted_centroid(q_content_ids, wn, corpus_idf, idf_floor) + if q_centroid is not None: + for mi, mem in enumerate(mems): + m_scoring_ids = self._get_mem_scoring_ids(mem) + m_centroid = self._compute_idf_weighted_centroid( + m_scoring_ids, wn, corpus_idf, idf_floor) + if m_centroid is not None: + centroid_scores[mi] = (q_centroid @ m_centroid).item() + diag.top_centroid_cosine = centroid_scores.max().item() if C_init > 0 else 0.0 + combined_sim = (self.c.ret_centroid_weight * centroid_scores + + self.c.ret_sem_weight * sem_sim_t + + self.c.ret_bidi_min_weight * bidi_min_t + + self.c.ret_forward_maxsim_weight * forward_t + + self.c.ret_dir_weight * raw_dir_sim) + C = C_init + top_sem = sem_sim_t.max().item() if C > 0 else 0.0 + top_bidi = bidi_min_t.max().item() if C > 0 else 0.0 + sem_thresh = max(self.c.gate_sem_floor, top_sem * self.c.gate_sem_ratio) + bidi_thresh = max(self.c.gate_bidi_floor, top_bidi * self.c.gate_bidi_ratio, + self.c.gate_bidi_hard_min) + hard_mask = (sem_sim_t >= sem_thresh) & (bidi_min_t >= bidi_thresh) + gate_affinity = (self.c.gate_sem_weight * sem_sim_t + + self.c.gate_bidi_weight * bidi_min_t) + diag.top_gate_affinity = gate_affinity.max().item() if C > 0 else 0.0 + diag.gate_threshold = max(sem_thresh, bidi_thresh) + diag.n_gate_pass = int(hard_mask.sum().item()) + if hard_mask.sum().item() == 0 and C > 0: + and_score = torch.minimum(sem_sim_t, bidi_min_t) + hard_mask[and_score.argmax()] = True + diag.n_after_hard_filter = int(hard_mask.sum().item()) + for mi, mem in enumerate(mems): + diag.per_memory_gate_affinity[mem.mid] = gate_affinity[mi].item() + keep_indices = hard_mask.nonzero(as_tuple=True)[0] + if keep_indices.numel() > 0 and keep_indices.numel() < C: + mems = [mems[i] for i in keep_indices.tolist()] + sb = sb[keep_indices]; sf = sf[keep_indices] + combined_sim = combined_sim[keep_indices] + raw_dir_sim = raw_dir_sim[keep_indices] + forward_t = forward_t[keep_indices]; bidi_min_t = bidi_min_t[keep_indices] + sem_sim_t = sem_sim_t[keep_indices]; centroid_scores = centroid_scores[keep_indices] + C = len(mems) + rerank_scores = self.reranker( + xq[b:b+1], fq[b:b+1], sb.unsqueeze(0), sf.unsqueeze(0), + combined_sim.unsqueeze(0)).squeeze(0) + diag.reranker_delta_mean = (rerank_scores - combined_sim).abs().mean().item() + diag.top_reranker_score = rerank_scores.max().item() if C > 0 else 0.0 + if C > 1: + top_score = rerank_scores.max() + score_mask = rerank_scores >= top_score * self.c.score_keep_ratio + if score_mask.sum().item() < 1: score_mask[rerank_scores.argmax()] = True + score_keep = score_mask.nonzero(as_tuple=True)[0] + diag.n_after_score_filter = score_keep.numel() + if score_keep.numel() < C: + mems = [mems[i] for i in score_keep.tolist()] + sb = sb[score_keep]; sf = sf[score_keep] + rerank_scores = rerank_scores[score_keep] + forward_t = forward_t[score_keep]; bidi_min_t = bidi_min_t[score_keep] + sem_sim_t = sem_sim_t[score_keep]; centroid_scores = centroid_scores[score_keep] + C = len(mems) + else: diag.n_after_score_filter = C + if C > 1 and forward_t.max().item() > 0: + top_fwd_here = forward_t.max() + coherence_mask = forward_t >= top_fwd_here * self.c.fwd_coherence_ratio + if coherence_mask.sum() >= 1: + coherence_keep = coherence_mask.nonzero(as_tuple=True)[0] + diag.n_after_coherence_filter = coherence_keep.numel() + if coherence_keep.numel() < C: + mems = [mems[i] for i in coherence_keep.tolist()] + sb = sb[coherence_keep]; sf = sf[coherence_keep] + rerank_scores = rerank_scores[coherence_keep] + forward_t = forward_t[coherence_keep]; bidi_min_t = bidi_min_t[coherence_keep] + sem_sim_t = sem_sim_t[coherence_keep]; centroid_scores = centroid_scores[coherence_keep] + C = len(mems) + else: diag.n_after_coherence_filter = C + else: diag.n_after_coherence_filter = C + if C > 1 and bidi_min_t.max().item() > 0: + top_bidi_here = bidi_min_t.max().item() + gap_mask = bidi_min_t >= (top_bidi_here - self.c.bidi_absolute_gap) + if gap_mask.sum() >= 1: + gap_keep = gap_mask.nonzero(as_tuple=True)[0] + diag.n_after_bidi_gap_filter = gap_keep.numel() + if gap_keep.numel() < C: + mems = [mems[i] for i in gap_keep.tolist()] + sb = sb[gap_keep]; sf = sf[gap_keep] + rerank_scores = rerank_scores[gap_keep] + forward_t = forward_t[gap_keep]; bidi_min_t = bidi_min_t[gap_keep] + sem_sim_t = sem_sim_t[gap_keep]; centroid_scores = centroid_scores[gap_keep] + C = len(mems) + else: diag.n_after_bidi_gap_filter = C + else: diag.n_after_bidi_gap_filter = C + raw_composite = (0.4 * centroid_scores + 0.4 * forward_t + + 0.15 * bidi_min_t + 0.05 * sem_sim_t.clamp(min=0)) + if self.c.use_mean_centered_scoring and C >= self.c.mc_require_min_candidates: + C_f = float(C); sum_raw = raw_composite.sum() + centered = (C_f / (C_f - 1.0)) * raw_composite - sum_raw / (C_f - 1.0) + for mi, mem in enumerate(mems): + diag.mean_center_raw_scores[mem.mid] = raw_composite[mi].item() + diag.mean_center_final_scores[mem.mid] = centered[mi].item() + keep_mask = centered > self.c.mc_keep_margin + n_pass = int(keep_mask.sum().item()) + if n_pass < self.c.mc_min_keep: + keep_n = max(self.c.mc_min_keep, 1) + top_keep = centered.topk(min(keep_n, C)).indices + keep_mask = torch.zeros(C, dtype=torch.bool, device=dev) + keep_mask[top_keep] = True + dropped_local = (~keep_mask).nonzero(as_tuple=True)[0].tolist() + if dropped_local: + diag.mean_center_applied = True + diag.mean_center_dropped_ids = [mems[i].mid for i in dropped_local] + keep_local = keep_mask.nonzero(as_tuple=True)[0] + if keep_local.numel() < C: + mems = [mems[i] for i in keep_local.tolist()] + sb = sb[keep_local]; sf = sf[keep_local] + rerank_scores = rerank_scores[keep_local] + forward_t = forward_t[keep_local]; bidi_min_t = bidi_min_t[keep_local] + sem_sim_t = sem_sim_t[keep_local]; centroid_scores = centroid_scores[keep_local] + raw_composite = raw_composite[keep_local] + C = len(mems) + diag.n_after_mean_center = C + dominant_mid = None; non_dominant_mids = []; non_dom_weights = {} + if C >= 1: + final_rank = (0.4 * rerank_scores + 0.4 * centroid_scores + 0.2 * forward_t) + dom_idx = int(final_rank.argmax().item()) + dominant_mid = mems[dom_idx].mid + if C > 1: + nd_idx = torch.tensor([i for i in range(C) if i != dom_idx], device=dev) + nd_scores = final_rank[nd_idx] + nd_w = F.softmax(nd_scores / self.c.retrieval_weight_temperature, dim=0) + for j, idx in enumerate(nd_idx.tolist()): + mid_j = mems[idx].mid + non_dominant_mids.append(mid_j) + non_dom_weights[mid_j] = nd_w[j].item() + if not self.training and C > topk: + _, top_idx = rerank_scores.topk(topk) + mems = [mems[i] for i in top_idx.cpu().tolist()] + sb = sb[top_idx]; sf = sf[top_idx] + rerank_scores = rerank_scores[top_idx] + forward_t = forward_t[top_idx]; bidi_min_t = bidi_min_t[top_idx] + sem_sim_t = sem_sim_t[top_idx]; centroid_scores = centroid_scores[top_idx] + C = topk + for mi, mem in enumerate(mems): + diag.per_memory_forward_maxsim[mem.mid] = forward_t[mi].item() + diag.per_memory_bidi_min[mem.mid] = bidi_min_t[mi].item() + diag.per_memory_sem_sim[mem.mid] = sem_sim_t[mi].item() + diag.per_memory_centroid_cosine[mem.mid] = centroid_scores[mi].item() + qp = xq[b].unsqueeze(0).expand(C, -1) + geo_r = self.geo.solve(sb, qp) + transported = self.trans(sf, geo_r.path) + if self.training: + ret_s = self.retention(sb, sf, + torch.tensor([m.surprise for m in mems], **_dev(xq)), + torch.tensor([self.time - m.last for m in mems], **_dev(xq)), + torch.tensor([m.cnt for m in mems], **_dev(xq))) + transported = transported * ret_s.unsqueeze(-1) + if update_stats: + for m in mems: m.last = self.time; m.cnt += 1 + final_scores = (0.4 * rerank_scores + 0.4 * centroid_scores + 0.2 * forward_t) + w = F.softmax(final_scores / self.c.retrieval_weight_temperature, dim=0) + fs = (transported * w.unsqueeze(-1)).sum(0) + batch_mw = [(m.mid, w[mi].item()) for mi, m in enumerate(mems)] + all_batch_mw.append(batch_mw) + all_dominant.append(dominant_mid); all_non_dominant.append(non_dominant_mids) + all_non_dom_weights.append(non_dom_weights) + all_results.append(transported); all_masks.append(torch.ones(C, **_dev(xq))) + all_biases.append(final_scores / self.c.tau); all_summaries.append(fs) + maxC = max(r.shape[0] for r in all_results) + padded = []; pm = []; pd = [] + for bi in range(B): + r, mk, db = all_results[bi], all_masks[bi], all_biases[bi]; gap = maxC - r.shape[0] + if gap > 0: + pr = self.empty_state(xq[bi:bi+1], fq[bi:bi+1]).expand(gap, -1) + r = torch.cat([r, pr if self.training else pr.detach()], 0) + mk = torch.cat([mk, torch.zeros(gap, **_dev(xq))]) + db = torch.cat([db, torch.full((gap,), -1e9, **_dev(xq))]) + padded.append(r); pm.append(mk); pd.append(db) + mf = torch.stack(padded); mem_mask = torch.stack(pm); dir_bias = torch.stack(pd) + fiber_summary = torch.stack(all_summaries) + diag.fiber_summary_norm = fiber_summary.norm().item() + diag.batch_mem_weights = all_batch_mw + diag.dominant_per_batch = all_dominant + diag.non_dominant_per_batch = all_non_dominant + diag.non_dominant_weights_per_batch = all_non_dom_weights + if diag.dominant_per_batch and diag.dominant_per_batch[0] is not None: + diag.dominant_memory_id = diag.dominant_per_batch[0] + refined = self.attn(fq, mf, mem_mask=mem_mask, dir_bias=dir_bias) + return refined, mem_mask, fiber_summary, diag + + def decay(self): + rm = [] + for mid, m in self.tree.store.items(): + dt = torch.tensor([self.time - m.last], **_dev(m.base)) + cnt = torch.tensor([m.cnt], **_dev(m.base)) + with torch.no_grad(): + sc = self.retention(m.base.unsqueeze(0), m.fiber.unsqueeze(0), + torch.tensor([m.surprise], **_dev(m.base)), dt, cnt).item() + if sc < self.c.retention_gc_threshold: rm.append(mid) + for i in rm: self.tree.remove(i) + return len(rm) + + def consolidate(self): + ms = list(self.tree.store.values()) + if len(ms) < 2: return 0 + merged = set() + for i in range(len(ms)): + if ms[i].mid in merged: continue + for j in range(i+1, len(ms)): + if ms[j].mid in merged: continue + d = self.metric.midpoint_approx_distance( + ms[i].base.unsqueeze(0), ms[j].base.unsqueeze(0)).item() + if d < self.c.consol_dist: + if not self._check_consolidation_compatible( + ms[i].content_token_ids, ms[j].content_token_ids): continue + wi, wj = ms[i].cnt+1, ms[j].cnt+1; t = wi+wj + nb = (ms[i].base*wi + ms[j].base*wj) / t + nf = (ms[i].fiber*wi + ms[j].fiber*wj) / t + nd = self._compute_dirn(nb, nf) + ms[i].base = nb.detach().clone(); ms[i].fiber = nf.detach().clone() + ms[i].dirn = nd.detach().clone(); ms[i].cnt += ms[j].cnt + ms[i].surprise = max(ms[i].surprise, ms[j].surprise); ms[i].version += 1 + if ms[j].source_text and not ms[i].source_text: + ms[i].source_text = ms[j].source_text + ms[i].content_token_ids = list(set(ms[i].content_token_ids + ms[j].content_token_ids)) + ms[i].expanded_content_ids = list(set(ms[i].expanded_content_ids + ms[j].expanded_content_ids)) + if ms[i].semantic_emb is not None and ms[j].semantic_emb is not None: + ms[i].semantic_emb = ((ms[i].semantic_emb*wi + ms[j].semantic_emb*wj) / t).detach().clone() + elif ms[j].semantic_emb is not None: ms[i].semantic_emb = ms[j].semantic_emb.clone() + merged.add(ms[j].mid) + for mid in merged: del self.tree.store[mid] + if merged: self.tree.rebuild() + return len(merged) + +@dataclass +class DecodeContext: + prefix_cond: torch.Tensor + prefix_uncond: Optional[torch.Tensor] + fiber_summary: torch.Tensor + diag: RetrievalDiag + content_bias: torch.Tensor + suppression_bias: torch.Tensor + vocab_bias: Optional[torch.Tensor] + +_PREFIX_META_ATTR = "_mem_decode_prompt_len" +_PREFIX_GUIDANCE_ACTIVE_ATTR = "_mem_guidance_active" +_PREFIX_CONTENT_BIAS_ATTR = "_mem_content_bias" +_PREFIX_SUPPRESSION_BIAS_ATTR = "_mem_suppression_bias" + +def _set_prefix_meta(prefix_tensor, prompt_len): + try: setattr(prefix_tensor, _PREFIX_META_ATTR, int(prompt_len)) + except Exception: pass + +def _get_prefix_meta(prefix_tensor): + return getattr(prefix_tensor, _PREFIX_META_ATTR, None) + +def _set_prefix_guidance(prefix_tensor, active: bool): + try: setattr(prefix_tensor, _PREFIX_GUIDANCE_ACTIVE_ATTR, bool(active)) + except Exception: pass + +def _get_prefix_guidance(prefix_tensor): + return getattr(prefix_tensor, _PREFIX_GUIDANCE_ACTIVE_ATTR, False) + +def _set_prefix_biases(prefix_tensor, content_bias, suppression_bias): + try: + setattr(prefix_tensor, _PREFIX_CONTENT_BIAS_ATTR, content_bias) + setattr(prefix_tensor, _PREFIX_SUPPRESSION_BIAS_ATTR, suppression_bias) + except Exception: pass + +class MemLLM(nn.Module): + def __init__(self, c): + super().__init__(); self.c = c + self.amm = AMM(c); self.bridge = EmbBridge(c) + self.semantic_probe = PrefixSemanticProbe(c.d_LLM, c.L_mem, c.d_F) + self.vocab_proj = MemoryVocabProjector(c.d_F, c.d_LLM) + self.layer_pool = None; self.backbone = None + self.tok = None; self._degen_guard = None; self.content_classifier = None + self._wte_neighbor_cache = None + self._wte_normed = None + self._filler_centroid = None + + def load(self, name=None, dtype_name=None): + name = name or self.c.llm_name + dtype_name = dtype_name or self.c.llm_dtype + self.backbone = LLMBackbone(name, dtype_name=dtype_name) + self.tok = self.backbone.tokenizer + self.c.d_LLM = self.backbone.d_model + self.c.vocab_size = self.backbone.vocab_size + dev = next(self.parameters()).device + if self.bridge.proj.fkv.out_features != 2 * self.c.d_LLM: + self.bridge = EmbBridge(self.c).to(dev) + self.semantic_probe = PrefixSemanticProbe(self.c.d_LLM, self.c.L_mem, self.c.d_F).to(dev) + self.vocab_proj = MemoryVocabProjector(self.c.d_F, self.c.d_LLM).to(dev) + self.layer_pool = AdaptiveLayerPool(self.backbone.n_layers + 1, self.c.d_LLM).to(dev) + self.content_classifier = ContentTokenClassifier( + self.tok, self.c, vocab_size=self.backbone.vocab_size) + self._degen_guard = DegenerationGuard(self.tok, self.c, self.content_classifier) + wte_fp32 = self.backbone.input_embedding_weight().to(dev) + self.bridge.aligner.calibrate(wte_fp32) + self._wte_normed = F.normalize(wte_fp32.detach(), dim=-1, eps=1e-8) + self.amm.wte_normed = self._wte_normed + self._build_wte_neighbor_cache() + self._compute_filler_centroid() + return self + + def _compute_filler_centroid(self): + if self.content_classifier is None or self.backbone is None: + self._filler_centroid = None; return + wte = self.backbone.input_embedding_weight().to(next(self.parameters()).device) + V = wte.shape[0] + filler_ids = sorted(self.content_classifier.filler_ids) + valid = [t for t in filler_ids if t < V] + if len(valid) < 3: + self._filler_centroid = None; return + filler_vecs = wte[torch.tensor(valid, device=wte.device)] + centroid = filler_vecs.mean(0) + self._filler_centroid = F.normalize(centroid, dim=-1, eps=1e-8) + + def _build_wte_neighbor_cache(self): + if self.backbone is None or self.content_classifier is None: return + V = self.backbone.vocab_size + if V > self.c.wte_neighbor_max_vocab: + self._wte_neighbor_cache = {} + print(f" [neighbor cache] vocab_size={V} > {self.c.wte_neighbor_max_vocab}, skip") + return + wte_n = self._wte_normed; cc = self.content_classifier + content_list = sorted(cc.content_ids) + valid = [t for t in content_list if t < wte_n.shape[0]] + self._wte_neighbor_cache = {} + K = self.c.wte_neighbor_k; thresh = self.c.wte_neighbor_threshold + batch_size = 500 + for start in range(0, len(valid), batch_size): + batch_ids = valid[start:start+batch_size] + batch_t = torch.tensor(batch_ids, device=wte_n.device) + batch_vecs = wte_n[batch_t] + sims = batch_vecs @ wte_n.T + topk_vals, topk_ids = sims.topk(K+1, dim=-1) + for i, tid in enumerate(batch_ids): + neighbors = [] + for v_val, nid in zip(topk_vals[i], topk_ids[i]): + nid_int = nid.item() + if nid_int == tid: continue + if v_val.item() >= thresh and nid_int in cc.content_ids: + neighbors.append(nid_int) + self._wte_neighbor_cache[tid] = neighbors + + def _expand_content_ids(self, content_ids): + if not self._wte_neighbor_cache: return content_ids + expanded = set(content_ids) + for tid in content_ids: + neighbors = self._wte_neighbor_cache.get(tid, []) + expanded.update(neighbors) + return list(expanded) + + def _check_guidance_active(self, diag) -> bool: + thresh = self.c.guidance_min_memory_weight + if not diag or not diag.batch_mem_weights: + return False + for mem_weights in diag.batch_mem_weights: + for mid, w in mem_weights: + if w > thresh and mid in self.amm.tree.store: + return True + return False + + def fwd(self, ids, mask, prefix=None): + out = self.backbone(ids, mask, prefix=prefix) + if (prefix is None or self.training or self.content_classifier is None): + return out + prompt_len = _get_prefix_meta(prefix) + if prompt_len is None: return out + step = int(ids.shape[1]) - int(prompt_len) + if step < 0: return out + + guidance_active = _get_prefix_guidance(prefix) + if not guidance_active: + return out + + logits = out['logits']; dev = logits.device + V_lg = logits.shape[-1] + last = logits[:, -1:, :].clone() + mod_last = False + + if (self.c.use_fwd_path_hard_mask + and self.c.use_early_content_starter_hard_mask + and step < self.c.early_starter_hard_mask_steps): + starter_mask = self.content_classifier.content_starter_mask(dev) + V = min(V_lg, starter_mask.shape[0]) + mask_val = float(self.c.fwd_path_hard_mask_value) + mask_bool = starter_mask[:V].bool().view(1, 1, V) + last_V = last[:, :, :V] + last[:, :, :V] = torch.where( + mask_bool, last_V, torch.full_like(last_V, mask_val)) + mod_last = True + + content_bias = getattr(prefix, _PREFIX_CONTENT_BIAS_ATTR, None) + suppression_bias = getattr(prefix, _PREFIX_SUPPRESSION_BIAS_ATTR, None) + if self.c.use_fwd_path_content_bias and (content_bias is not None or suppression_bias is not None): + logits_std = logits.std().item() + dampen = self.c.fwd_path_bias_dampen + + if content_bias is not None: + step_scale = max(self.c.content_bias_floor, + 1.0 - step * self.c.content_bias_decay) + unit = (logits_std * self.c.content_bias_std_multiplier + if self.c.use_adaptive_content_bias_scale else 1.0) + V = min(V_lg, content_bias.shape[-1]) + cb = content_bias[:, :V].to(dev) + scale = unit * self.c.content_bias_scale * step_scale * dampen + last[:, 0, :V] = last[:, 0, :V] + cb * scale + mod_last = True + + if suppression_bias is not None and self.c.use_memory_guided_suppression: + step_scale_sup = max(self.c.suppression_floor, + 1.0 - step * self.c.suppression_decay) + unit_sup = (logits_std * self.c.suppression_std_multiplier + if self.c.use_adaptive_content_bias_scale else 1.0) + V = min(V_lg, suppression_bias.shape[-1]) + sb = suppression_bias[:, :V].to(dev) + scale_sup = unit_sup * self.c.suppression_bias_scale * step_scale_sup * dampen + last[:, 0, :V] = last[:, 0, :V] - sb * scale_sup + mod_last = True + + if self.c.use_no_repeat_bigram and step >= 2: + B = ids.shape[0] + pen = self.c.no_repeat_bigram_penalty + for b in range(B): + gen_ids_b = ids[b, int(prompt_len):].tolist() + if len(gen_ids_b) < 2: continue + last_tok = gen_ids_b[-1] + penalize_nexts = set() + for i in range(len(gen_ids_b) - 1): + if gen_ids_b[i] == last_tok: + penalize_nexts.add(gen_ids_b[i + 1]) + if penalize_nexts: + pen_ids = [t for t in penalize_nexts if 0 <= t < V_lg] + if pen_ids: + pen_t = torch.tensor(pen_ids, device=dev, dtype=torch.long) + last[b, 0, pen_t] = last[b, 0, pen_t] - pen + mod_last = True + + if mod_last: + new_logits = logits.clone() + new_logits[:, -1:, :] = last + out['logits'] = new_logits + return out + + def _compute_content_semantic_emb(self, hidden_states, ids, mask): + B, T, D = hidden_states.shape + cc = self.content_classifier + result = [] + for b in range(B): + content_positions = [] + T_valid = min(T, ids.shape[1]) if ids is not None else T + for pos in range(T_valid): + if mask is not None and mask.shape[1] > pos and mask[b, pos].item() == 0: + continue + if ids is not None: + tid = ids[b, pos].item() + if cc is not None and tid in cc.content_ids: + content_positions.append(min(pos, T-1)) + if content_positions: + pos_t = torch.tensor(content_positions, device=hidden_states.device) + content_hs = hidden_states[b, pos_t] + result.append(content_hs.mean(0)) + else: + if mask is not None: + valid_len = min(int(mask[b].sum().item()), T); valid_len = max(valid_len, 1) + result.append(hidden_states[b, :valid_len].mean(0)) + else: result.append(hidden_states[b].mean(0)) + return torch.stack(result) + + def extract_state(self, hs, mask=None, pl=0): + pooled = self.layer_pool(hs) + if pl > 0: pooled = pooled[:, pl:] + m = mask[:, pl:] if mask is not None and pl > 0 else mask + if m is not None and m.shape[1] != pooled.shape[1]: m = None + xq, fq = self.bridge.ext(pooled, m) + return pooled, xq, fq + + def _build_token_bias_from_memories(self, mem_weight_list, q_content_ids): + V = self.c.vocab_size; dev = next(self.parameters()).device + cc = self.content_classifier; wte_n = self._wte_normed + floor = self.c.content_bias_relevance_floor + concentration = self.c.content_bias_concentration + bias = torch.zeros(V, device=dev) + q_valid = [i for i in q_content_ids if i < wte_n.shape[0]] + q_vecs = wte_n[q_valid] if q_valid else None + for mid, weight in mem_weight_list: + if mid not in self.amm.tree.store or weight <= 0: continue + mem = self.amm.tree.store[mid] + scoring_ids = self.amm._get_mem_scoring_ids(mem) + if cc is not None and self.c.use_word_starter_filter: + valid_ids = [t for t in scoring_ids if t < V and t < wte_n.shape[0] + and t in cc.content_starter_ids] + elif cc is not None: + valid_ids = [t for t in scoring_ids if t < V and t < wte_n.shape[0] + and t in cc.content_ids] + else: valid_ids = [] + if not valid_ids: continue + if q_valid and q_vecs is not None: + m_vecs = wte_n[valid_ids]; sim = m_vecs @ q_vecs.T + relevance = sim.max(dim=1).values.clamp(min=0) + relevance = relevance.pow(concentration) + relevance = relevance * (1.0 - floor) + floor + for i, tid in enumerate(valid_ids): + bias[tid] += weight * relevance[i].item() + else: + for tid in valid_ids: bias[tid] += weight + return bias + + def _build_content_bias(self, diag, query_content_ids_per_batch): + V = self.c.vocab_size; dev = next(self.parameters()).device + B = len(diag.batch_mem_weights) + bias = torch.zeros(B, V, device=dev) + for b, mem_weights in enumerate(diag.batch_mem_weights): + q_ids = (query_content_ids_per_batch[b] + if query_content_ids_per_batch and b < len(query_content_ids_per_batch) + else []) + reweighted = [(mid, w * (diag.per_memory_bidi_min.get(mid, 0.5) ** 2)) + for mid, w in mem_weights] + b_bias = self._build_token_bias_from_memories(reweighted, q_ids) + bmax = b_bias.max() + if bmax > 1e-8: bias[b] = b_bias / bmax + return bias + + def _build_suppression_bias(self, diag, query_content_ids_per_batch): + V = self.c.vocab_size; dev = next(self.parameters()).device + B = len(diag.batch_mem_weights) + suppression = torch.zeros(B, V, device=dev) + cc = self.content_classifier + if cc is None: return suppression + for b in range(B): + dom_mid = diag.dominant_per_batch[b] if b < len(diag.dominant_per_batch) else None + nd_mids = (diag.non_dominant_per_batch[b] + if b < len(diag.non_dominant_per_batch) else []) + nd_weights = (diag.non_dominant_weights_per_batch[b] + if b < len(diag.non_dominant_weights_per_batch) else {}) + if not nd_mids: continue + dom_token_set = set() + if dom_mid is not None and dom_mid in self.amm.tree.store: + dom_mem = self.amm.tree.store[dom_mid] + for t in self.amm._get_mem_scoring_ids(dom_mem): + if t in cc.content_ids: dom_token_set.add(t) + q_ids = (query_content_ids_per_batch[b] + if query_content_ids_per_batch and b < len(query_content_ids_per_batch) + else []) + nd_mem_weights = [(mid, nd_weights.get(mid, 0.0)) for mid in nd_mids] + nd_bias = self._build_token_bias_from_memories(nd_mem_weights, q_ids) + for t in dom_token_set: + if 0 <= t < V: nd_bias[t] = 0.0 + nmax = nd_bias.max() + if nmax > 1e-8: suppression[b] = nd_bias / nmax + return suppression + + def _get_prefix(self, hs, mask=None, pl=0, update_stats=True, return_extra=False, ids=None): + pooled, xq, fq = self.extract_state(hs, mask, pl) + trimmed_mask = mask[:, pl:] if mask is not None and pl > 0 else mask + if trimmed_mask is not None and pooled.shape[1] != trimmed_mask.shape[1]: + trimmed_mask = None + query_content_ids_per_batch = [] + if ids is not None and self.content_classifier is not None: + for b in range(ids.shape[0]): + b_ids = ids[b].tolist() + b_exact = list(set(self.content_classifier.get_content_ids_from_tokens(b_ids))) + query_content_ids_per_batch.append(b_exact) + query_sem = (self._compute_content_semantic_emb(pooled, ids, trimmed_mask) + if ids is not None and self.content_classifier is not None + else pooled.mean(1)) + wte_n = self._wte_normed + fibers, mem_mask, fiber_summary, diag = self.amm.retrieve_multi( + xq, fq, update_stats=update_stats, + query_semantic_emb=query_sem, + query_content_ids_per_batch=query_content_ids_per_batch, + wte_normed=wte_n, content_classifier=self.content_classifier) + prefix = self.bridge.inject( + fibers, mem_mask, fiber_summary=fiber_summary, + filler_centroid=self._filler_centroid) + + prompt_len_for_meta = (mask.shape[1] if mask is not None + else (ids.shape[1] if ids is not None else hs.shape[1])) + _set_prefix_meta(prefix, prompt_len_for_meta) + + if return_extra: + # ctx-path: shape_step_logits handles all shaping. + # fwd() must be a pure backbone pass → guidance=False, no biases attached. + _set_prefix_guidance(prefix, False) + content_bias = self._build_content_bias(diag, query_content_ids_per_batch) + suppression_bias = (self._build_suppression_bias(diag, query_content_ids_per_batch) + if self.c.use_memory_guided_suppression + else torch.zeros_like(content_bias)) + return prefix, fiber_summary, diag, content_bias, suppression_bias + + # Runner-direct path: gate on actual retrieval content. + if not self.training: + guidance = self._check_guidance_active(diag) + _set_prefix_guidance(prefix, guidance) + if self.c.use_fwd_path_content_bias and guidance: + with torch.no_grad(): + cb = self._build_content_bias(diag, query_content_ids_per_batch) + sb = (self._build_suppression_bias(diag, query_content_ids_per_batch) + if self.c.use_memory_guided_suppression else None) + _set_prefix_biases(prefix, cb, sb) + return prefix + + def _build_contrastive_uncond_prefix(self, diag, prefix_cond, prompt_len_for_meta=None): + dev = prefix_cond.device; B = prefix_cond.shape[0] + non_dom_fibers = []; have_contrast = [] + for b in range(B): + mids = diag.non_dominant_per_batch[b] if b < len(diag.non_dominant_per_batch) else [] + mids = [m for m in mids if m in self.amm.tree.store] + if mids: + fvecs = torch.stack([self.amm.tree.store[m].fiber.to(dev) for m in mids]) + non_dom_fibers.append(fvecs.mean(0)); have_contrast.append(True) + else: + non_dom_fibers.append(torch.zeros(self.c.d_F, device=dev)); have_contrast.append(False) + non_dom_fibers_t = torch.stack(non_dom_fibers, dim=0) + uncond_prefix = torch.zeros_like(prefix_cond) + for b in range(B): + if have_contrast[b]: + single = non_dom_fibers_t[b:b+1].unsqueeze(1) + mask_one = torch.ones(1, 1, device=dev) + pref_b = self.bridge.inject( + single, mask_one, fiber_summary=non_dom_fibers_t[b:b+1], + filler_centroid=self._filler_centroid) + uncond_prefix[b:b+1] = pref_b + else: + uncond_prefix[b:b+1] = self.bridge.build_neutral_prefix(1, dev) + if prompt_len_for_meta is not None: + _set_prefix_meta(uncond_prefix, prompt_len_for_meta) + # CFG contrast branch: fwd() must not apply shaping. + _set_prefix_guidance(uncond_prefix, False) + return uncond_prefix + + def _compute_vocab_bias(self, fiber_summary): + if fiber_summary is None: return None + wte = self.backbone.input_embedding_weight().to(fiber_summary.device) + return self.vocab_proj(fiber_summary, wte) + + def prepare_decode_context(self, ids, mask, update_stats=True): + prompt_len = ids.shape[1] + with torch.no_grad(): + o = self.fwd(ids, mask) + prefix_cond, fs, diag, cb, sb = self._get_prefix( + o['hs'], mask, update_stats=update_stats, return_extra=True, ids=ids) + vb = self._compute_vocab_bias(fs) + if self.c.use_cfg_decoding: + if self.c.use_contrastive_memory_cfg: + prefix_uncond = self._build_contrastive_uncond_prefix( + diag, prefix_cond, prompt_len_for_meta=prompt_len) + else: + B = prefix_cond.shape[0] + prefix_uncond = self.bridge.build_neutral_prefix(B, prefix_cond.device) + _set_prefix_meta(prefix_uncond, prompt_len) + _set_prefix_guidance(prefix_uncond, False) + else: + prefix_uncond = None + return DecodeContext( + prefix_cond=prefix_cond, prefix_uncond=prefix_uncond, + fiber_summary=fs, diag=diag, + content_bias=cb, suppression_bias=sb, vocab_bias=vb) + + def shape_step_logits(self, logits_cond, logits_uncond, step, + content_bias, suppression_bias, vocab_bias, state): + c = self.c; dev = logits_cond.device; cc = self.content_classifier + HARD_MASK = -1e9 + if c.use_cfg_decoding and logits_uncond is not None: + alpha = c.cfg_scale + if c.cfg_decay_steps > 0: + alpha *= max(0.0, 1.0 - step / c.cfg_decay_steps) + lg = logits_cond + alpha * (logits_cond - logits_uncond) + else: + lg = logits_cond.clone() + V_lg = lg.shape[-1] + if c.use_adaptive_content_bias_scale: + logits_std = lg.std().item() + cb_unit = logits_std * c.content_bias_std_multiplier + sup_unit = logits_std * c.suppression_std_multiplier + else: + cb_unit = 1.0; sup_unit = 1.0 + step_scale_cb = max(c.content_bias_floor, 1.0 - step * c.content_bias_decay) + if content_bias is not None and content_bias.abs().max().item() > 0.01: + V = min(V_lg, content_bias.shape[-1]) + lg[:, :V] = lg[:, :V] + content_bias[:, :V] * cb_unit * c.content_bias_scale * step_scale_cb + step_scale_sup = max(c.suppression_floor, 1.0 - step * c.suppression_decay) + if (c.use_memory_guided_suppression and suppression_bias is not None + and suppression_bias.abs().max().item() > 0.01): + V = min(V_lg, suppression_bias.shape[-1]) + lg[:, :V] = lg[:, :V] - suppression_bias[:, :V] * sup_unit * c.suppression_bias_scale * step_scale_sup + step_scale_learned = max(c.semantic_boost_floor, 1.0 - step * c.semantic_boost_decay) + if vocab_bias is not None: + V2 = min(V_lg, vocab_bias.shape[-1]) + lg[:, :V2] = lg[:, :V2] + vocab_bias[:, :V2] * c.semantic_boost_scale * step_scale_learned + if cc: + for tid, count in state.generated_content_counts.items(): + if tid in cc.content_ids and tid < V_lg: + scaled_count = count ** c.content_repeat_exponent + lg[0, tid] -= c.content_repeat_penalty * scaled_count + if c.use_cyclic_content_hard_mask and cc is not None: + window = c.cyclic_content_window; max_cnt = c.cyclic_content_max_count + window_counts = {}; cutoff_step = step - window + for (step_idx, tid) in state.content_history: + if step_idx >= cutoff_step: + window_counts[tid] = window_counts.get(tid, 0) + 1 + for tid, cnt in window_counts.items(): + if cnt >= max_cnt and 0 <= tid < V_lg: + lg[0, tid] = HARD_MASK + if c.use_ngram_repeat_block and len(state.generated_ids) >= 4: + max_n = min(c.ngram_repeat_max_n, len(state.generated_ids) // 2) + for n in range(2, max_n + 1): + if len(state.generated_ids) >= 2 * n: + tail = state.generated_ids[-n:] + prev = state.generated_ids[-2 * n:-n] + if tail == prev: + expected_next = state.generated_ids[-n] + if 0 <= expected_next < V_lg: + lg[0, expected_next] -= c.ngram_repeat_penalty + + if c.use_no_repeat_bigram and len(state.generated_ids) >= 2: + last_tok = state.generated_ids[-1] + penalize_nexts = set() + for i in range(len(state.generated_ids) - 1): + if state.generated_ids[i] == last_tok: + penalize_nexts.add(state.generated_ids[i + 1]) + for next_tok in penalize_nexts: + if 0 <= next_tok < V_lg: + lg[0, next_tok] -= c.no_repeat_bigram_penalty + + if cc and self._wte_neighbor_cache and state.recent_starters: + for prev_tid, _ in state.recent_starters: + neighbors = self._wte_neighbor_cache.get(prev_tid, []) + for nid in neighbors: + if nid in cc.word_starter_ids: continue + if nid < V_lg: lg[0, nid] -= c.bpe_echo_penalty + if cc and state.generated_ids and state.generated_ids[-1] in cc.content_starter_ids: + for tid in cc.content_ids: + if tid not in cc.word_starter_ids and tid < V_lg: + lg[0, tid] -= c.post_starter_nonstarter_penalty + newline_ids_set = cc.newline_ids if cc is not None else set() + if c.use_newline_hard_gate and cc is not None: + content_count_so_far = sum(state.generated_content_counts.values()) + hard_gate_active = (step < c.newline_hard_gate_min_step + or content_count_so_far < c.newline_hard_gate_min_content) + if hard_gate_active: + for nid in newline_ids_set: + if nid < V_lg: lg[0, nid] = HARD_MASK + eos_token_id = self.tok.eos_token_id + if (c.use_eos_hard_mask and eos_token_id is not None + and step < c.eos_hard_mask_steps and eos_token_id < V_lg): + lg[0, eos_token_id] = HARD_MASK + if c.use_content_gated_newline and cc is not None: + content_count_so_far = sum(state.generated_content_counts.values()) + if content_count_so_far < c.min_content_tokens_before_newline: + for nid in newline_ids_set: + if nid < V_lg: lg[0, nid] -= c.late_newline_penalty + if (c.use_early_content_starter_hard_mask and cc is not None + and step < c.early_starter_hard_mask_steps): + starter_mask = cc.content_starter_mask(dev)[:V_lg] + lg[0, :V_lg] = torch.where( + starter_mask.bool(), lg[0, :V_lg], + torch.full_like(lg[0, :V_lg], HARD_MASK)) + if self._degen_guard is not None: + lg = self._degen_guard.process(lg, state.generated_ids, step) + return lg + + def write(self, text, training_mode=False): + tk = self.tok(text, return_tensors='pt', padding=True, truncation=True) + ids, mask = tk['input_ids'], tk['attention_mask'] + dev = next(self.parameters()).device; ids, mask = ids.to(dev), mask.to(dev) + with torch.no_grad(): + o = self.fwd(ids, mask) + hs_pooled = self.layer_pool(o['hs']) + surp = self.amm.surprise_proxy(o['logits'][:, :-1], ids[:, 1:]) + pooled_mean = hs_pooled.mean(1) + content_sem = self._compute_content_semantic_emb(hs_pooled, ids, mask) + raw_ids = self.tok.encode(text); cc = self.content_classifier + content_ids = list(set(cc.get_content_ids_from_tokens(raw_ids))) if cc else [] + expanded_ids = self._expand_content_ids(content_ids) + stored = 0; gate_vals = [] + for b in range(ids.shape[0]): + with torch.no_grad(): + gate = self.amm.write_gate(pooled_mean[b:b+1], surp[b:b+1]).item() + gate_vals.append(gate) + if training_mode or gate >= self.c.write_gate_threshold: + self.amm.store_mem(pooled_mean[b], surp[b], training_mode, + source_text=text, content_token_ids=content_ids, + content_semantic_emb=content_sem[b], + expanded_content_ids=expanded_ids) + stored += 1 + return stored, gate_vals + + def _refresh_all_memories(self): + entries = list(self.amm.tree.store.values()) + texts = [e.source_text for e in entries if e.source_text] + if not texts: return 0 + unique_texts = list(dict.fromkeys(texts)) + self.amm.tree.store.clear() + self.amm.tree.root = _Node() + self.amm.tree.nid = 0; self.amm.time = 0 + for text in unique_texts: self.write(text, training_mode=True) + return len(unique_texts) + + def _prep_prompt_ids(self, prompt): + if self.c.use_chat_template_for_gen and self.backbone.has_chat_template: + prompt = self.backbone.build_chat_text(prompt) + tk = self.tok(prompt, return_tensors='pt') + return tk['input_ids'], tk['attention_mask'] + + def generate(self, prompt, mt=50, greedy=False): + ids, mask = self._prep_prompt_ids(prompt) + dev = next(self.parameters()).device + ids = ids.to(dev); mask = mask.to(dev) + ctx = self.prepare_decode_context(ids, mask, update_stats=True) + state = DecodeState(); prompt_len = ids.shape[1] + for i in range(mt): + if i > 0 and i % self.c.retrieval_interval == 0: + ctx = self.prepare_decode_context(ids, mask, update_stats=True) + with torch.no_grad(): + o_cond = self.fwd(ids, mask, ctx.prefix_cond) + lg_cond = o_cond['logits'][:, -1:].squeeze(1) + if self.c.use_cfg_decoding and ctx.prefix_uncond is not None: + o_uncond = self.fwd(ids, mask, ctx.prefix_uncond) + lg_uncond = o_uncond['logits'][:, -1:].squeeze(1) + else: + lg_uncond = None + lg = self.shape_step_logits(lg_cond, lg_uncond, i, + ctx.content_bias, ctx.suppression_bias, ctx.vocab_bias, state) + if greedy: + nxt = lg.argmax(-1, keepdim=True) + else: + lg_t = lg / self.c.gen_temp; p = F.softmax(lg_t, -1) + sp, si = torch.sort(p, descending=True); cs = torch.cumsum(sp, -1) + rm = cs - sp > self.c.gen_top_p; sp[rm] = 0 + total = sp.sum(-1, keepdim=True) + if (total < 1e-10).any(): sp[:, 0] = 1.0; total = sp.sum(-1, keepdim=True) + sp = sp / total; nxt = si.gather(-1, torch.multinomial(sp, 1)) + nxt_id = nxt.item() + if nxt_id == self.tok.eos_token_id and len(state.generated_ids) >= self.c.degen_min_tokens: + break + state.update(nxt_id, i, self.content_classifier, + self.c.bpe_echo_window, self.c.cyclic_content_window) + ids = torch.cat([ids, nxt], 1) + mask = torch.cat([mask, torch.ones(1, 1, device=dev, dtype=mask.dtype)], 1) + new_ids = ids[0, prompt_len:].tolist() + gen_text = self.tok.decode(new_ids, skip_special_tokens=True) + return prompt + gen_text if not self.c.use_chat_template_for_gen else gen_text + + def save_memory(self, path): + data = {'store': {}, 'nid': self.amm.tree.nid, 'time': self.amm.time} + for mid, m in self.amm.tree.store.items(): + data['store'][mid] = { + 'base': m.base.cpu(), 'fiber': m.fiber.cpu(), 'dirn': m.dirn.cpu(), + 'surprise': m.surprise, 'ts': m.ts, 'last': m.last, 'cnt': m.cnt, 'version': m.version, + 'source_text': m.source_text, + 'content_token_ids': m.content_token_ids, + 'expanded_content_ids': m.expanded_content_ids, + 'semantic_emb': m.semantic_emb.cpu() if m.semantic_emb is not None else None} + torch.save(data, path) + + def load_memory(self, path): + data = torch.load(path, weights_only=False) + self.amm.tree.store.clear(); self.amm.tree.root = _Node() + self.amm.tree.nid = data['nid']; self.amm.time = data['time'] + dev = next(self.parameters()).device + for mid, d in data['store'].items(): + sem = d.get('semantic_emb', None) + if sem is not None: sem = sem.to(dev) + m = MemEntry(mid=mid, base=d['base'].to(dev), fiber=d['fiber'].to(dev), + dirn=d['dirn'].to(dev), surprise=d['surprise'], ts=d['ts'], + last=d['last'], cnt=d['cnt'], version=d['version'], + source_text=d.get('source_text', ''), + content_token_ids=d.get('content_token_ids', []), + expanded_content_ids=d.get('expanded_content_ids', []), + semantic_emb=sem) + self.amm.tree.insert(m) + +class Trainer: + def __init__(self, m, c): + self.m = m; self.c = c + ps = [p for n, p in m.named_parameters() if p.requires_grad and 'backbone' not in n] + self.opt = torch.optim.AdamW(ps, lr=1e-4, weight_decay=0.01) + self.warmup = LossWarmup({ + 'semantic_probe': c.warmup_steps_probe, 'dir_diversity': c.warmup_steps_dd, + 'reranker_ranking': c.warmup_steps_rr, 'vocab_anchor': c.warmup_steps_va, + 'semantic_alignment': c.warmup_steps_sa, + 'tail_semantic_anchor': c.warmup_steps_tsa}) + self.grad_monitor = GradientMonitor() + self.grad_monitor.register('ctx_encoder', m.amm.ctx) + self.grad_monitor.register('fib_encoder', m.amm.fib) + self.grad_monitor.register('dir_predictor', m.amm.dir_pred) + self.grad_monitor.register('fiber_connection', m.amm.conn) + self.grad_monitor.register('fiber_attn', m.amm.attn) + self.grad_monitor.register('reranker', m.amm.reranker) + self.grad_monitor.register('qformer', m.bridge.proj) + self.grad_monitor.register('content_bypass', m.bridge.bypass) + self.grad_monitor.register('semantic_probe', m.semantic_probe) + self.grad_monitor.register('layer_pool', m.layer_pool) + self.grad_monitor.register('prefix_aligner', m.bridge.aligner) + self.grad_monitor.register('vocab_proj', m.vocab_proj) + if c.use_content_semantic_tail and c.content_tail_slots > 0: + self.grad_monitor.register('tail_head', m.bridge.tail_head) + self.layer_weight_history = []; self._step_count = 0 + + def _encode_with_grad(self, texts): + tk = self.m.tok(texts, return_tensors='pt', padding=True, truncation=True) + dev = next(self.m.parameters()).device + ids, mask = tk['input_ids'].to(dev), tk['attention_mask'].to(dev) + with torch.no_grad(): + o = self.m.fwd(ids, mask) + surp = self.m.amm.surprise_proxy(o['logits'][:, :-1], ids[:, 1:]) + pooled = self.m.layer_pool(o['hs']); pooled_mean = pooled.mean(1) + base = self.m.amm.ctx(pooled_mean) + fiber = self.m.amm.fib(pooled_mean, base, surp) + _ = self.m.amm.dir_pred(base, fiber) + return ids, mask, base, fiber, surp, pooled_mean + + def encoder_throughput_loss(self, ids, mask, fiber): + B = ids.shape[0]; dev = ids.device + fiber_unsq = fiber.unsqueeze(1); mem_mask_ones = torch.ones(B, 1, device=dev) + prefix = self.m.bridge.inject(fiber_unsq, mem_mask_ones, fiber_summary=fiber) + o2 = self.m.fwd(ids, mask, prefix) + lg = o2['logits'][:, o2['pl']:-1]; tg = ids[:, 1:] + ml = min(lg.shape[1], tg.shape[1]) + if ml == 0: return torch.tensor(0.0, device=dev, requires_grad=True) + return F.cross_entropy(lg[:, :ml].reshape(-1, lg.shape[-1]), tg[:, :ml].reshape(-1)) + + def semantic_alignment_loss(self, fiber, target_ids, target_mask): + dev = fiber.device + wte = self.m.backbone.input_embedding_weight().to(dev) + vocab_logits = self.m.vocab_proj(fiber, wte) + B, V = vocab_logits.shape; cc = self.m.content_classifier + if cc is None: return torch.tensor(0.0, device=dev, requires_grad=True) + target = torch.zeros(B, V, device=dev); valid_count = 0 + for b in range(B): + valid = target_ids[b][target_mask[b].bool()].tolist() + content_ids = cc.get_content_ids_from_tokens(valid) + if content_ids: + uids = list(set(content_ids)); uids = [uid for uid in uids if uid < V] + if uids: target[b, uids] = 1.0 / len(uids); valid_count += 1 + if valid_count == 0: return torch.tensor(0.0, device=dev, requires_grad=True) + log_probs = F.log_softmax(vocab_logits / self.c.semantic_align_temp, dim=-1) + kl = F.kl_div(log_probs, target, reduction='none').sum(-1) + return kl.mean() + + def vocab_anchor_loss(self, prefix): + dev = prefix.device + wte = self.m.backbone.input_embedding_weight().to(dev) + pn = F.normalize(prefix.reshape(-1, prefix.shape[-1]), dim=-1) + wn = F.normalize(wte, dim=-1) + sim = pn @ wn.T; topk_sim = sim.topk(self.c.vocab_anchor_topk, dim=-1).values + return -topk_sim.mean() + + def tail_semantic_anchor_loss(self, fiber, ids, mask): + if not (self.c.use_content_semantic_tail and self.c.content_tail_slots > 0): + return torch.tensor(0.0, device=fiber.device, requires_grad=True) + tail = self.m.bridge.tail_head(fiber) + if tail is None: + return torch.tensor(0.0, device=fiber.device, requires_grad=True) + dev = fiber.device + wte = self.m.backbone.input_embedding_weight().to(dev) + B, n_slots, _ = tail.shape; V = wte.shape[0] + cc = self.m.content_classifier + if cc is None: return torch.tensor(0.0, device=dev, requires_grad=True) + losses = [] + tn = F.normalize(tail, dim=-1); wn = F.normalize(wte, dim=-1) + for b in range(B): + valid = ids[b][mask[b].bool()].tolist() + content_tids = list(set(cc.get_content_ids_from_tokens(valid))) + content_tids = [t for t in content_tids if t < V] + if not content_tids: continue + target = torch.zeros(V, device=dev) + target[content_tids] = 1.0 / len(content_tids) + slot_logits = tn[b] @ wn.T / 0.3 + log_probs = F.log_softmax(slot_logits, dim=-1) + kl = F.kl_div(log_probs, target.unsqueeze(0).expand_as(log_probs), + reduction='none').sum(-1).mean() + losses.append(kl) + if not losses: + return torch.tensor(0.0, device=dev, requires_grad=True) + return torch.stack(losses).mean() + + def _recon_forward(self, text): + tk = self.m.tok(text, return_tensors='pt', padding=True, truncation=True) + dev = next(self.m.parameters()).device + ids, mask = tk['input_ids'].to(dev), tk['attention_mask'].to(dev) + with torch.no_grad(): bo = self.m.fwd(ids, mask) + prefix = self.m._get_prefix(bo['hs'], mask, update_stats=False, ids=ids) + o = self.m.fwd(ids, mask, prefix) + lg = o['logits'][:, o['pl']:-1]; tg = ids[:, 1:] + ml = min(lg.shape[1], tg.shape[1]) + if ml == 0: + zero = ids.new_tensor(0.0, dtype=torch.float, requires_grad=True) + return zero, prefix, self.m.bridge._last_fiber_summary + l_r = F.cross_entropy(lg[:, :ml].reshape(-1, lg.shape[-1]), tg[:, :ml].reshape(-1)) + fs = self.m.bridge._last_fiber_summary + if fs is None: fs = torch.zeros(1, self.c.d_F, device=dev) + return l_r, prefix, fs + + def recon(self, text): + loss, prefix, fs = self._recon_forward(text) + return {'loss': loss, 'prefix': prefix, 'fiber_summary': fs} + + def _semantic_probe_loss(self, prefix_batch, fs_batch): + pred = self.m.semantic_probe(prefix_batch) + l_mse = F.mse_loss(pred, fs_batch.detach()) + if prefix_batch.shape[0] >= 2: + pn = F.normalize(pred, dim=-1); tn = F.normalize(fs_batch.detach(), dim=-1) + sim = pn @ tn.T / self.c.probe_contrastive_tau + lb = torch.arange(prefix_batch.shape[0], device=prefix_batch.device) + l_ctr = F.cross_entropy(sim, lb) + return l_mse + 0.5 * l_ctr + return l_mse + + def contrast(self, texts): + tk = self.m.tok(texts, return_tensors='pt', padding=True, truncation=True) + dev = next(self.m.parameters()).device + ids, mask = tk['input_ids'].to(dev), tk['attention_mask'].to(dev) + with torch.no_grad(): o = self.m.fwd(ids, mask) + _, xq, fq = self.m.extract_state(o['hs'], mask) + x = F.normalize(self.m.amm.contrast_proj_x(xq), -1) + f = F.normalize(self.m.amm.contrast_proj_f(fq), -1) + sxf = x @ f.T / self.c.contrast_tau; sfx = f @ x.T / self.c.contrast_tau + lb = torch.arange(len(texts), device=dev) + return (F.cross_entropy(sxf, lb) + F.cross_entropy(sfx, lb)) / 2 + + def holonomy_proxy(self, x, f): + sz = 0.05; v1 = torch.randn_like(x) * sz; v2 = torch.randn_like(x) * sz + loop = torch.stack([x, x+v1, x+v1+v2, x+v2, x], 1) + return (self.m.amm.trans(f, loop) - f).pow(2).sum(-1).mean() + + def write_policy_loss(self, texts): + tk = self.m.tok(texts, return_tensors='pt', padding=True, truncation=True) + dev = next(self.m.parameters()).device + ids, mask = tk['input_ids'].to(dev), tk['attention_mask'].to(dev) + with torch.no_grad(): + o = self.m.fwd(ids, mask) + surp = self.m.amm.surprise_proxy(o['logits'][:, :-1], ids[:, 1:]) + pooled = self.m.layer_pool(o['hs']).mean(1) + gates = self.m.amm.write_gate(pooled, surp) + labels = (surp > surp.median()).float() + return F.binary_cross_entropy(gates, labels) + + def direction_diversity_loss(self, texts): + tk = self.m.tok(texts, return_tensors='pt', padding=True, truncation=True) + dev = next(self.m.parameters()).device + ids, mask = tk['input_ids'].to(dev), tk['attention_mask'].to(dev) + with torch.no_grad(): o = self.m.fwd(ids, mask) + _, xq, fq = self.m.extract_state(o['hs'], mask) + dirs = F.normalize(self.m.amm.dir_pred(xq, fq), dim=-1, eps=1e-8) + dir_sim = (dirs @ dirs.T).clamp(-1.0, 1.0) + with torch.no_grad(): + fn = F.normalize(fq, dim=-1, eps=1e-8); fiber_sim = (fn @ fn.T).clamp(-1.0, 1.0) + tau = self.c.dir_diversity_tau + dir_prob = torch.sigmoid(dir_sim / tau); fiber_prob = torch.sigmoid(fiber_sim / tau) + B = len(texts); mask_off = ~torch.eye(B, dtype=torch.bool, device=dev) + return F.binary_cross_entropy(dir_prob[mask_off], fiber_prob[mask_off].detach()) + + def reranker_ranking_loss(self, texts): + store = self.m.amm.tree.store + if len(store) < 2: + dev = next(self.m.parameters()).device + return torch.tensor(0.0, device=dev, requires_grad=True) + tk = self.m.tok(texts, return_tensors='pt', padding=True, truncation=True) + dev = next(self.m.parameters()).device + ids, mask = tk['input_ids'].to(dev), tk['attention_mask'].to(dev) + with torch.no_grad(): o = self.m.fwd(ids, mask) + _, xq, fq = self.m.extract_state(o['hs'], mask) + mids = list(store.keys()) + cb = torch.stack([store[m].base.to(dev) for m in mids]) + cf = torch.stack([store[m].fiber.to(dev) for m in mids]) + cd = torch.stack([store[m].dirn.to(dev) for m in mids]) + B = xq.shape[0]; qdir = self.m.amm.dir_pred(xq, fq) + dir_sims = torch.einsum('bd,cd->bc', qdir, cd) + cb_e = cb.unsqueeze(0).expand(B, -1, -1); cf_e = cf.unsqueeze(0).expand(B, -1, -1) + scores = self.m.amm.reranker(xq, fq, cb_e, cf_e, dir_sims) + with torch.no_grad(): + fqn = F.normalize(fq, dim=-1); cfn = F.normalize(cf, dim=-1) + relevance = torch.einsum('bd,cd->bc', fqn, cfn) + s_mean = scores.mean(-1, keepdim=True); s_std = scores.std(-1, keepdim=True).clamp(min=1e-6) + r_mean = relevance.mean(-1, keepdim=True); r_std = relevance.std(-1, keepdim=True).clamp(min=1e-6) + sn = (scores - s_mean) / s_std; rn = (relevance - r_mean) / r_std + return F.mse_loss(sn, rn.detach()) + + def step(self, texts): + self.m.train(); self.opt.zero_grad() + dev = next(self.m.parameters()).device; W = self.c.loss_weights + ids_enc, mask_enc, base, fiber, surp, pooled_mean = self._encode_with_grad(texts) + l_et = self.encoder_throughput_loss(ids_enc, mask_enc, fiber) + w_sa = self.warmup.weight('semantic_alignment') + l_sa = self.semantic_alignment_loss(fiber, ids_enc, mask_enc) * w_sa + w_tsa = self.warmup.weight('tail_semantic_anchor') + l_tsa = self.tail_semantic_anchor_loss(fiber, ids_enc, mask_enc) * w_tsa + all_lr = []; all_pf = []; all_fs = [] + for t in texts: + r = self.recon(t) + all_lr.append(r['loss']); all_pf.append(r['prefix']) + fs = r['fiber_summary'] + all_fs.append(fs if fs is not None else torch.zeros(1, self.c.d_F, device=dev)) + l_r = sum(all_lr) / len(texts) + pf_batch = torch.cat(all_pf, 0); fs_batch = torch.cat(all_fs, 0) + w_sp = self.warmup.weight('semantic_probe') + l_sp = self._semantic_probe_loss(pf_batch, fs_batch) * w_sp + w_va = self.warmup.weight('vocab_anchor') + l_va = self.vocab_anchor_loss(pf_batch) * w_va + l_c = self.contrast(texts) if len(texts) >= 2 else torch.tensor(0.0, device=dev) + with torch.no_grad(): + tk2 = self.m.tok(texts, return_tensors='pt', padding=True, truncation=True) + ids2, mask2 = tk2['input_ids'].to(dev), tk2['attention_mask'].to(dev) + o2 = self.m.fwd(ids2, mask2) + _, xq2, fq2 = self.m.extract_state(o2['hs'], mask2) + l_h = self.holonomy_proxy(xq2, fq2) + l_w = self.write_policy_loss(texts) + w_dd = self.warmup.weight('dir_diversity') + l_dd = (self.direction_diversity_loss(texts) if len(texts) >= 2 + else torch.tensor(0.0, device=dev)) * w_dd + w_rr = self.warmup.weight('reranker_ranking') + l_rr = self.reranker_ranking_loss(texts) * w_rr + loss = (W['recon']*l_r + W['semantic_alignment']*l_sa + + W['encoder_throughput']*l_et + W['contrast']*l_c + + W['holonomy']*l_h + W['write_policy']*l_w + + W['semantic_probe']*l_sp + W['dir_diversity']*l_dd + + W['reranker_ranking']*l_rr + W['vocab_anchor']*l_va + + W.get('tail_semantic_anchor', 0.5)*l_tsa) + loss.backward() + nn.utils.clip_grad_norm_( + [p for n, p in self.m.named_parameters() + if p.requires_grad and 'backbone' not in n], 1.) + self.opt.step(); self.warmup.advance(); self._step_count += 1 + grad_norms = self.grad_monitor.snapshot() + self.layer_weight_history.append(self.m.layer_pool.weight_dist().cpu().numpy().copy()) + if self._step_count % self.c.refresh_memories_every == 0: + self.m.eval() + with torch.no_grad(): self.m._refresh_all_memories() + self.m.train() + self.m.eval() + return {'total': loss.item(), 'recon': l_r.item(), 'contrast': l_c.item(), + 'holonomy': l_h.item(), 'write_policy': l_w.item(), + 'semantic_probe': l_sp.item(), 'dir_diversity': l_dd.item(), + 'reranker_ranking': l_rr.item(), 'encoder_throughput': l_et.item(), + 'vocab_anchor': l_va.item(), 'semantic_alignment': l_sa.item(), + 'tail_semantic_anchor': l_tsa.item(), + 'grad_norms': grad_norms, 'loss_weights': W}