diff --git a/curricula/cs336_a1/modules/adamw/bugs/missing_bias_correction.json b/curricula/cs336_a1/modules/adamw/bugs/missing_bias_correction.json index bfdcb28c80..d9d2e88f15 100644 --- a/curricula/cs336_a1/modules/adamw/bugs/missing_bias_correction.json +++ b/curricula/cs336_a1/modules/adamw/bugs/missing_bias_correction.json @@ -1,67 +1,26 @@ { "id": "adamw-missing-bias-correction", - "description": "AdamW optimizer missing bias correction", + "description": "Use the raw first moment without bias correction (exp_avg_hat = exp_avg).", "injection_type": "ast", "engine_version": "2.1", - "target_function": "adamw", + "target_function": "step", "logic": [ { "pass": 1, "type": "find_and_replace", - "description": "Delete bias_correction1 calculation", "pattern": { "node_type": "Assign", - "targets": [{"node_type": "Name", "id": "bias_correction1"}], - "value": {"node_type": "BinOp", "op": "Sub"} - }, - "replacement": {"type": "delete_statement"} - }, - { - "pass": 2, - "type": "find_and_replace", - "description": "Delete bias_correction2 calculation", - "pattern": { - "node_type": "Assign", - "targets": [{"node_type": "Name", "id": "bias_correction2"}], - "value": {"node_type": "BinOp", "op": "Sub"} - }, - "replacement": {"type": "delete_statement"} - }, - { - "pass": 3, - "type": "find_and_replace", - "description": "Simplify step_size calculation (remove bias correction)", - "pattern": { - "node_type": "Assign", - "targets": [{"node_type": "Name", "id": "step_size"}], - "value": {"node_type": "BinOp", "op": "Div"} - }, - "replacement": { - "type": "replace_value_with", - "source": "lr" - } - }, - { - "pass": 4, - "type": "find_and_replace", - "description": "Simplify denominator (remove bias correction)", - "pattern": { - "node_type": "Assign", - "targets": [{"node_type": "Name", "id": "denom"}], - "value": {"node_type": "Call"} + "targets": [ + { + "node_type": "Name", + "id": "exp_avg_hat" + } + ] }, "replacement": { "type": "replace_value_with", - "source": "exp_avg_sq.sqrt().add_(eps)" + "source": "exp_avg" } } - ], - "metadata": { - "created": "2025-11-13", - "version": "2.1", - "author": "golden_example", - "symptom": "Without bias correction, AdamW updates are biased toward zero (10× too small at step 1), severely slowing initial training", - "complexity": "complex", - "operations": 4 - } + ] } diff --git a/curricula/cs336_a1/modules/adamw/bugs/missing_bias_correction.patch b/curricula/cs336_a1/modules/adamw/bugs/missing_bias_correction.patch deleted file mode 100644 index da8a45d739..0000000000 --- a/curricula/cs336_a1/modules/adamw/bugs/missing_bias_correction.patch +++ /dev/null @@ -1,19 +0,0 @@ ---- a/cs336_basics/optimizer.py -+++ b/cs336_basics/optimizer.py -@@ -85,10 +85,11 @@ class AdamW(torch.optim.Optimizer): - exp_avg.mul_(beta1).add_(grad, alpha=1 - beta1) - exp_avg_sq.mul_(beta2).addcmul_(grad, grad, value=1 - beta2) - -- # Bias correction -- bias_correction1 = 1 - beta1 ** state['step'] -- bias_correction2 = 1 - beta2 ** state['step'] -- step_size = lr / bias_correction1 -+ # BUG: Missing bias correction! Without this, early updates are biased -+ # toward zero (10× too small at step 1), severely slowing initial training. -+ # Should compute: bias_correction1 = 1 - beta1 ** state['step'] -+ # bias_correction2 = 1 - beta2 ** state['step'] -+ step_size = lr # Should be: lr / bias_correction1 - - # Compute update -- denom = (exp_avg_sq.sqrt() / math.sqrt(bias_correction2)).add_(eps) -+ denom = exp_avg_sq.sqrt().add_(eps) # Should divide by sqrt(bias_correction2) diff --git a/curricula/cs336_a1/modules/adamw/bugs/missing_bias_correction_symptom.txt b/curricula/cs336_a1/modules/adamw/bugs/missing_bias_correction_symptom.txt new file mode 100644 index 0000000000..c15322bde4 --- /dev/null +++ b/curricula/cs336_a1/modules/adamw/bugs/missing_bias_correction_symptom.txt @@ -0,0 +1,14 @@ +# Bug Symptom: Adamw — adamw-missing-bias-correction + +## Observed Behavior +A subtle semantic bug has been injected into your `step` implementation. Your code +runs without raising a syntax error, but the module's correctness test now fails: +Use the raw first moment without bias correction (exp_avg_hat = exp_avg). + +## Your Challenge +A single line in `cs336_basics/optimizer.py` no longer matches your correct implementation. Read the +function, form a hypothesis from the symptom above, and locate the divergence. +Restore the correct behavior — do not rewrite the whole function. + +## Verify +Run `mastery submit` to re-run the validator once you believe the bug is fixed. diff --git a/curricula/cs336_a1/modules/attention/bugs/missing_scale.json b/curricula/cs336_a1/modules/attention/bugs/missing_scale.json index acc5e3196f..756ff6b739 100644 --- a/curricula/cs336_a1/modules/attention/bugs/missing_scale.json +++ b/curricula/cs336_a1/modules/attention/bugs/missing_scale.json @@ -1,6 +1,6 @@ { "id": "attention-missing-scale", - "description": "Missing scaling by sqrt(d_k) in attention scores, causing training instability", + "description": "Drop the 1/sqrt(d_k) scaling factor (scale set to 1.0).", "injection_type": "ast", "engine_version": "2.1", "target_function": "scaled_dot_product_attention", @@ -8,68 +8,19 @@ { "pass": 1, "type": "find_and_replace", - "description": "Delete the d_k variable assignment", "pattern": { "node_type": "Assign", "targets": [ { "node_type": "Name", - "id": "d_k" + "id": "scale" } - ], - "value": { - "node_type": "Subscript", - "value": { - "node_type": "Attribute", - "value": { - "node_type": "Name", - "id": "Q" - }, - "attr": "shape" - } - } - }, - "replacement": { - "type": "delete_statement" - } - }, - { - "pass": 2, - "type": "find_and_replace", - "description": "Delete the scores scaling assignment", - "pattern": { - "node_type": "Assign", - "targets": [ - { - "node_type": "Name", - "id": "scores" - } - ], - "value": { - "node_type": "BinOp", - "op": "Div", - "left": { - "node_type": "Name", - "id": "scores" - }, - "right": { - "node_type": "Call", - "func": { - "node_type": "Attribute", - "attr": "sqrt" - } - } - } + ] }, "replacement": { - "type": "delete_statement" + "type": "replace_value_with", + "source": "1.0" } } - ], - "metadata": { - "created": "2025-11-13", - "version": "2.1", - "author": "Manual (Golden Example)", - "tier": "simple" - } + ] } diff --git a/curricula/cs336_a1/modules/attention/bugs/missing_scale.patch b/curricula/cs336_a1/modules/attention/bugs/missing_scale.patch deleted file mode 100644 index ad720dfd21..0000000000 --- a/curricula/cs336_a1/modules/attention/bugs/missing_scale.patch +++ /dev/null @@ -1,15 +0,0 @@ ---- a/cs336_basics/layers.py -+++ b/cs336_basics/layers.py -@@ -180,8 +180,9 @@ def scaled_dot_product_attention( - # Compute attention scores: Q @ K^T - scores = Q @ K.transpose(-2, -1) - -- # Scale by sqrt(d_k) for stability -- d_k = Q.shape[-1] -- scores = scores / math.sqrt(d_k) -+ # BUG: Missing scaling by sqrt(d_k)! -+ # Without scaling, large dot products push softmax into saturated regions -+ # causing vanishing gradients and training instability. -+ # Should be: scores = scores / math.sqrt(Q.shape[-1]) - - # Apply causal mask if provided diff --git a/curricula/cs336_a1/modules/attention/bugs/missing_scale_symptom.txt b/curricula/cs336_a1/modules/attention/bugs/missing_scale_symptom.txt new file mode 100644 index 0000000000..5b31012c2b --- /dev/null +++ b/curricula/cs336_a1/modules/attention/bugs/missing_scale_symptom.txt @@ -0,0 +1,14 @@ +# Bug Symptom: Attention — attention-missing-scale + +## Observed Behavior +A subtle semantic bug has been injected into your `scaled_dot_product_attention` implementation. Your code +runs without raising a syntax error, but the module's correctness test now fails: +Drop the 1/sqrt(d_k) scaling factor (scale set to 1.0). + +## Your Challenge +A single line in `cs336_basics/layers.py` no longer matches your correct implementation. Read the +function, form a hypothesis from the symptom above, and locate the divergence. +Restore the correct behavior — do not rewrite the whole function. + +## Verify +Run `mastery submit` to re-run the validator once you believe the bug is fixed. diff --git a/curricula/cs336_a1/modules/bpe_tokenizer/bugs/wrong_merge_order.json b/curricula/cs336_a1/modules/bpe_tokenizer/bugs/wrong_merge_order.json index 27b957616d..8dd9d50534 100644 --- a/curricula/cs336_a1/modules/bpe_tokenizer/bugs/wrong_merge_order.json +++ b/curricula/cs336_a1/modules/bpe_tokenizer/bugs/wrong_merge_order.json @@ -1,22 +1,28 @@ { - "id": "bpe-tokenizer-wrong-merge-order", - "description": "Merges inserted at beginning instead of appended", + "id": "bpe-tokenizer-dropped-right-byte", + "description": "A merged token keeps only the left byte sequence and drops the right, so the learned vocabulary diverges from a correct BPE merge.", "injection_type": "ast", "engine_version": "2.1", - "target_function": "train", - "logic": [{ - "pass": 1, - "type": "find_and_replace", - "pattern": { - "node_type": "Expr", - "value": { - "node_type": "Call", - "func": {"node_type": "Attribute", "attr": "append"} + "target_function": "train_bpe", + "logic": [ + { + "pass": 1, + "type": "find_and_replace", + "pattern": { + "node_type": "Assign", + "value": { + "node_type": "BinOp", + "op": "Add", + "right": { + "node_type": "Name", + "id": "right_bytes" + } + } + }, + "replacement": { + "type": "replace_value_with", + "source": "left_bytes" } - }, - "replacement": { - "type": "replace_with", - "source": "merges.insert(0, (vocab[best_pair[0]], vocab[best_pair[1]]))" } - }] -} \ No newline at end of file + ] +} diff --git a/curricula/cs336_a1/modules/bpe_tokenizer/bugs/wrong_merge_order.patch b/curricula/cs336_a1/modules/bpe_tokenizer/bugs/wrong_merge_order.patch deleted file mode 100644 index 334b4ad067..0000000000 --- a/curricula/cs336_a1/modules/bpe_tokenizer/bugs/wrong_merge_order.patch +++ /dev/null @@ -1,15 +0,0 @@ ---- a/cs336_basics/tokenizer.py -+++ b/cs336_basics/tokenizer.py -@@ -45,8 +45,10 @@ def train_bpe(text, vocab_size): - # Find most frequent pair - best_pair = max(pair_counts, key=pair_counts.get) - -- # Record merge (in order!) -- merges.append((vocab[best_pair[0]], vocab[best_pair[1]])) -+ # BUG: Recording merges in wrong order! Inserting at beginning instead of appending. -+ # This reverses merge order, breaking encoding since later merges depend on earlier ones. -+ # Should be: merges.append(...) -+ merges.insert(0, (vocab[best_pair[0]], vocab[best_pair[1]])) # Wrong! - - # Create new token - vocab[token_id] = vocab[best_pair[0]] + vocab[best_pair[1]] diff --git a/curricula/cs336_a1/modules/bpe_tokenizer/bugs/wrong_merge_order_draft.json b/curricula/cs336_a1/modules/bpe_tokenizer/bugs/wrong_merge_order_draft.json deleted file mode 100644 index 94764ff5d8..0000000000 --- a/curricula/cs336_a1/modules/bpe_tokenizer/bugs/wrong_merge_order_draft.json +++ /dev/null @@ -1,53 +0,0 @@ -{ - "id": "bpe-tokenizer-wrong-merge-order", - "description": "Reverses merge order by inserting at the beginning instead of appending, breaking encoding since later merges depend on earlier ones.", - "injection_type": "ast", - "engine_version": "2.1", - "target_function": "bpe_tokenizer", - "logic": [ - { - "pass_": 1, - "type": "find_and_replace", - "description": "Find merges.append(...) and replace with merges.insert(0, ...) to reverse the order.", - "pattern": { - "node_type": "Expr", - "targets": null, - "value": { - "node_type": "Call", - "op": null, - "attr": null - }, - "attr": null, - "op": null, - "func": { - "node_type": "Attribute", - "op": null, - "attr": "append" - }, - "left": null, - "right": null, - "args": [ - { - "node_type": "Tuple", - "op": null, - "attr": null - } - ], - "keywords": null - }, - "conditions": null, - "track_as": null, - "replacement": { - "type": "replace_with", - "source": "node", - "name": null - } - } - ], - "metadata": { - "created": "2023-10-05", - "version": "2.0", - "author": "LLM-Generated", - "tier": "simple" - } -} \ No newline at end of file diff --git a/curricula/cs336_a1/modules/bpe_tokenizer/bugs/wrong_merge_order_symptom.txt b/curricula/cs336_a1/modules/bpe_tokenizer/bugs/wrong_merge_order_symptom.txt new file mode 100644 index 0000000000..c9c52a9a87 --- /dev/null +++ b/curricula/cs336_a1/modules/bpe_tokenizer/bugs/wrong_merge_order_symptom.txt @@ -0,0 +1,14 @@ +# Bug Symptom: Bpe Tokenizer — bpe-tokenizer-wrong-merge-count + +## Observed Behavior +A subtle semantic bug has been injected into your `train_bpe` implementation. Your code +runs without raising a syntax error, but the module's correctness test now fails: +A merged token keeps only the left byte sequence and drops the right, so the learned vocabulary diverges from a correct BPE merge. + +## Your Challenge +A single line in `cs336_basics/bpe.py` no longer matches your correct implementation. Read the +function, form a hypothesis from the symptom above, and locate the divergence. +Restore the correct behavior — do not rewrite the whole function. + +## Verify +Run `mastery submit` to re-run the validator once you believe the bug is fixed. diff --git a/curricula/cs336_a1/modules/bpe_tokenizer/validator.sh b/curricula/cs336_a1/modules/bpe_tokenizer/validator.sh index 1198209ea4..930c43eea9 100755 --- a/curricula/cs336_a1/modules/bpe_tokenizer/validator.sh +++ b/curricula/cs336_a1/modules/bpe_tokenizer/validator.sh @@ -15,7 +15,7 @@ fi # HARDEN STAGE: File already copied by submit-fix, just cd to shadow worktree if [ "$(pwd)" != "$SHADOW_WORKTREE" ]; then # BUILD STAGE: We're in main directory, copy file and cd to shadow worktree - cp cs336_basics/tokenizer.py "$SHADOW_WORKTREE/cs336_basics/tokenizer.py" + cp cs336_basics/bpe.py "$SHADOW_WORKTREE/cs336_basics/bpe.py" cd "$SHADOW_WORKTREE" else # HARDEN STAGE: Already in shadow worktree, file was copied by submit-fix @@ -30,14 +30,14 @@ start_time=$(python3 -c 'import time; print(time.time())') if [ -n "$MASTERY_PYTHON" ]; then # Engine provided its Python executable - use it export PYTHONPATH="$(pwd):$PYTHONPATH" - "$MASTERY_PYTHON" -m pytest tests/test_tokenizer.py::test_train_bpe -v --tb=short --import-mode=importlib + "$MASTERY_PYTHON" -m pytest tests/test_train_bpe.py::test_train_bpe -v --tb=short --import-mode=importlib elif [ -n "$VIRTUAL_ENV" ]; then # We're in an active virtual environment - use its Python explicitly export PYTHONPATH="$(pwd):$PYTHONPATH" - "$VIRTUAL_ENV/bin/python" -m pytest tests/test_tokenizer.py::test_train_bpe -v --tb=short --import-mode=importlib + "$VIRTUAL_ENV/bin/python" -m pytest tests/test_train_bpe.py::test_train_bpe -v --tb=short --import-mode=importlib else # No active environment - use uv to create one - uv run pytest tests/test_tokenizer.py::test_train_bpe -v --tb=short --import-mode=importlib + uv run pytest tests/test_train_bpe.py::test_train_bpe -v --tb=short --import-mode=importlib fi # Record end time diff --git a/curricula/cs336_a1/modules/checkpointing/bugs/missing_optimizer_state.json b/curricula/cs336_a1/modules/checkpointing/bugs/missing_optimizer_state.json index 303d810971..07fd83f7a3 100644 --- a/curricula/cs336_a1/modules/checkpointing/bugs/missing_optimizer_state.json +++ b/curricula/cs336_a1/modules/checkpointing/bugs/missing_optimizer_state.json @@ -1,6 +1,6 @@ { "id": "checkpointing-missing-optimizer-state", - "description": "Missing optimizer state from checkpoint", + "description": "Checkpoint omits optimizer_state_dict, so resume loses optimizer moments.", "injection_type": "ast", "engine_version": "2.1", "target_function": "save_checkpoint", @@ -13,12 +13,9 @@ "targets": [ { "node_type": "Name", - "id": "checkpoint" + "id": "payload" } - ], - "value": { - "node_type": "Dict" - } + ] }, "replacement": { "type": "replace_value_with", @@ -26,4 +23,4 @@ } } ] -} \ No newline at end of file +} diff --git a/curricula/cs336_a1/modules/checkpointing/bugs/missing_optimizer_state.patch b/curricula/cs336_a1/modules/checkpointing/bugs/missing_optimizer_state.patch deleted file mode 100644 index 04a2213572..0000000000 --- a/curricula/cs336_a1/modules/checkpointing/bugs/missing_optimizer_state.patch +++ /dev/null @@ -1,23 +0,0 @@ ---- a/cs336_basics/utils.py -+++ b/cs336_basics/utils.py -@@ -5,9 +5,10 @@ def save_checkpoint(model, optimizer, iteration, out): - """Save complete training state.""" - checkpoint = { - 'model_state_dict': model.state_dict(), -- 'optimizer_state_dict': optimizer.state_dict(), - 'iteration': int(iteration), - } -+ # BUG: Missing optimizer state! -+ # Without optimizer momentum, training restarts poorly - - torch.save(checkpoint, out) - -@@ -16,7 +17,8 @@ def load_checkpoint(src, model, optimizer): - checkpoint = torch.load(src, map_location='cpu') - - model.load_state_dict(checkpoint['model_state_dict']) -- optimizer.load_state_dict(checkpoint['optimizer_state_dict']) -+ # BUG: Can't load optimizer state - wasn't saved! -+ # KeyError: 'optimizer_state_dict' - - return int(checkpoint['iteration']) diff --git a/curricula/cs336_a1/modules/checkpointing/bugs/missing_optimizer_state_draft.json b/curricula/cs336_a1/modules/checkpointing/bugs/missing_optimizer_state_draft.json deleted file mode 100644 index 4820a4a707..0000000000 --- a/curricula/cs336_a1/modules/checkpointing/bugs/missing_optimizer_state_draft.json +++ /dev/null @@ -1,102 +0,0 @@ -{ - "id": "checkpointing-missing-optimizer-state", - "description": "Removes the optimizer state from the checkpoint, causing training to restart poorly.", - "injection_type": "ast", - "engine_version": "2.1", - "target_function": "save_checkpoint", - "logic": [ - { - "pass_": 1, - "type": "find_and_replace", - "description": "Delete the optimizer state from the checkpoint dictionary.", - "pattern": { - "node_type": "Assign", - "targets": [ - { - "node_type": "Name", - "id": "checkpoint" - } - ], - "value": { - "node_type": "Dict", - "op": null, - "attr": null - }, - "attr": null, - "op": null, - "func": null, - "left": null, - "right": null, - "args": null, - "keywords": [ - { - "arg": "optimizer_state_dict", - "value": { - "node_type": "Call", - "op": null, - "attr": null - } - } - ] - }, - "conditions": null, - "track_as": null, - "replacement": { - "type": "replace_value_with", - "source": "{\"model_state_dict\": model.state_dict(), \"iteration\": int(iteration)}", - "name": null - } - }, - { - "pass_": 2, - "type": "find_and_replace", - "description": "Remove the optimizer state loading from the checkpoint.", - "pattern": { - "node_type": "Expr", - "targets": null, - "value": { - "node_type": "Call", - "op": null, - "attr": null - }, - "attr": null, - "op": null, - "func": { - "node_type": "Attribute", - "op": null, - "attr": "load_state_dict" - }, - "left": null, - "right": null, - "args": [ - { - "node_type": "Subscript", - "op": null, - "attr": null - } - ], - "keywords": null - }, - "conditions": [ - { - "check": "target_is_name", - "value": null, - "index": 0, - "name": "optimizer" - } - ], - "track_as": null, - "replacement": { - "type": "delete_statement", - "source": null, - "name": null - } - } - ], - "metadata": { - "created": "2025-11-13", - "version": "2.0", - "author": "LLM-Generated", - "tier": "simple" - } -} \ No newline at end of file diff --git a/curricula/cs336_a1/modules/checkpointing/bugs/missing_optimizer_state_symptom.txt b/curricula/cs336_a1/modules/checkpointing/bugs/missing_optimizer_state_symptom.txt new file mode 100644 index 0000000000..9bde20a653 --- /dev/null +++ b/curricula/cs336_a1/modules/checkpointing/bugs/missing_optimizer_state_symptom.txt @@ -0,0 +1,14 @@ +# Bug Symptom: Checkpointing — checkpointing-missing-optimizer-state + +## Observed Behavior +A subtle semantic bug has been injected into your `save_checkpoint` implementation. Your code +runs without raising a syntax error, but the module's correctness test now fails: +Checkpoint omits optimizer_state_dict, so resume loses optimizer moments. + +## Your Challenge +A single line in `cs336_basics/utils.py` no longer matches your correct implementation. Read the +function, form a hypothesis from the symptom above, and locate the divergence. +Restore the correct behavior — do not rewrite the whole function. + +## Verify +Run `mastery submit` to re-run the validator once you believe the bug is fixed. diff --git a/curricula/cs336_a1/modules/checkpointing/validator.sh b/curricula/cs336_a1/modules/checkpointing/validator.sh index f8ea992057..8690e2aedf 100755 --- a/curricula/cs336_a1/modules/checkpointing/validator.sh +++ b/curricula/cs336_a1/modules/checkpointing/validator.sh @@ -20,12 +20,12 @@ start_time=$(python3 -c 'import time; print(time.time())') if [ -n "$MASTERY_PYTHON" ]; then export PYTHONPATH="$(pwd):$PYTHONPATH" - "$MASTERY_PYTHON" -m pytest tests/test_training.py::test_checkpoint -v --tb=short --import-mode=importlib + "$MASTERY_PYTHON" -m pytest tests/test_serialization.py::test_checkpointing -v --tb=short --import-mode=importlib elif [ -n "$VIRTUAL_ENV" ]; then export PYTHONPATH="$(pwd):$PYTHONPATH" - "$VIRTUAL_ENV/bin/python" -m pytest tests/test_training.py::test_checkpoint -v --tb=short --import-mode=importlib + "$VIRTUAL_ENV/bin/python" -m pytest tests/test_serialization.py::test_checkpointing -v --tb=short --import-mode=importlib else - uv run pytest tests/test_training.py::test_checkpoint -v --tb=short --import-mode=importlib + uv run pytest tests/test_serialization.py::test_checkpointing -v --tb=short --import-mode=importlib fi end_time=$(python3 -c 'import time; print(time.time())') diff --git a/curricula/cs336_a1/modules/cosine_schedule/bugs/wrong_cosine_range.json b/curricula/cs336_a1/modules/cosine_schedule/bugs/wrong_cosine_range.json index bada3177b8..5b4a57fa05 100644 --- a/curricula/cs336_a1/modules/cosine_schedule/bugs/wrong_cosine_range.json +++ b/curricula/cs336_a1/modules/cosine_schedule/bugs/wrong_cosine_range.json @@ -1,9 +1,9 @@ { "id": "cosine-schedule-wrong-range", - "description": "Cosine goes negative without proper transformation", + "description": "Cosine progress omits the warmup offset (it / denom), shifting the schedule.", "injection_type": "ast", "engine_version": "2.1", - "target_function": "cosine_learning_rate_schedule", + "target_function": "get_lr_cosine_schedule", "logic": [ { "pass": 1, @@ -13,17 +13,14 @@ "targets": [ { "node_type": "Name", - "id": "cosine_decay" + "id": "progress" } - ], - "value": { - "node_type": "Call" - } + ] }, "replacement": { "type": "replace_value_with", - "source": "math.cos(math.pi * progress)" + "source": "it / denom" } } ] -} \ No newline at end of file +} diff --git a/curricula/cs336_a1/modules/cosine_schedule/bugs/wrong_cosine_range.patch b/curricula/cs336_a1/modules/cosine_schedule/bugs/wrong_cosine_range.patch deleted file mode 100644 index bbcd9e5c17..0000000000 --- a/curricula/cs336_a1/modules/cosine_schedule/bugs/wrong_cosine_range.patch +++ /dev/null @@ -1,16 +0,0 @@ ---- a/cs336_basics/optimizer.py -+++ b/cs336_basics/optimizer.py -@@ -45,9 +45,10 @@ def lr_cosine_schedule( - # Cosine decay phase - progress = (step - warmup_steps) / (max_steps - warmup_steps) - -- # Apply cosine formula: (1 + cos(πt)) / 2 maps [-1,1] to [0,1] -- cosine_decay = 0.5 * (1.0 + math.cos(math.pi * progress)) -- lr = min_lr + (max_lr - min_lr) * cosine_decay -+ # BUG: Using cos(πt) directly without transformation! -+ # This goes negative after t=0.5, making learning rate negative. -+ # Should be: (1 + cos(πt)) / 2 to map [-1,1] → [0,1] -+ cosine_decay = math.cos(math.pi * progress) # Wrong! Goes from 1 to -1 -+ lr = min_lr + (max_lr - min_lr) * cosine_decay # Will be negative! - - return lr diff --git a/curricula/cs336_a1/modules/cosine_schedule/bugs/wrong_cosine_range_draft.json b/curricula/cs336_a1/modules/cosine_schedule/bugs/wrong_cosine_range_draft.json deleted file mode 100644 index f8aff672e7..0000000000 --- a/curricula/cs336_a1/modules/cosine_schedule/bugs/wrong_cosine_range_draft.json +++ /dev/null @@ -1,58 +0,0 @@ -{ - "id": "cosine_schedule-bug", - "description": "Bug in cosine_schedule where cosine decay is used directly without transformation, causing learning rate to become negative.", - "injection_type": "ast", - "engine_version": "2.1", - "target_function": "cosine_schedule", - "logic": [ - { - "pass_": 1, - "type": "find_and_replace", - "description": "Replace cosine decay calculation to include transformation (1 + cos(\u03c0t)) / 2.", - "pattern": { - "node_type": "Assign", - "targets": [ - { - "node_type": "Name", - "id": "cosine_decay" - } - ], - "value": { - "node_type": "Call", - "op": null, - "attr": null - }, - "attr": null, - "op": null, - "func": { - "node_type": "Attribute", - "op": null, - "attr": "cos" - }, - "left": null, - "right": null, - "args": [ - { - "node_type": "BinOp", - "op": "Mult", - "attr": null - } - ], - "keywords": null - }, - "conditions": null, - "track_as": null, - "replacement": { - "type": "replace_value_with", - "source": "0.5 * (1.0 + math.cos(math.pi * progress))", - "name": null - } - } - ], - "metadata": { - "created": "2023-11-19", - "version": "2.0", - "author": "LLM-Generated", - "tier": "simple" - } -} \ No newline at end of file diff --git a/curricula/cs336_a1/modules/cosine_schedule/bugs/wrong_cosine_range_symptom.txt b/curricula/cs336_a1/modules/cosine_schedule/bugs/wrong_cosine_range_symptom.txt new file mode 100644 index 0000000000..3eee781734 --- /dev/null +++ b/curricula/cs336_a1/modules/cosine_schedule/bugs/wrong_cosine_range_symptom.txt @@ -0,0 +1,14 @@ +# Bug Symptom: Cosine Schedule — cosine-schedule-wrong-range + +## Observed Behavior +A subtle semantic bug has been injected into your `get_lr_cosine_schedule` implementation. Your code +runs without raising a syntax error, but the module's correctness test now fails: +Cosine progress omits the warmup offset (it / denom), shifting the schedule. + +## Your Challenge +A single line in `cs336_basics/utils.py` no longer matches your correct implementation. Read the +function, form a hypothesis from the symptom above, and locate the divergence. +Restore the correct behavior — do not rewrite the whole function. + +## Verify +Run `mastery submit` to re-run the validator once you believe the bug is fixed. diff --git a/curricula/cs336_a1/modules/cosine_schedule/validator.sh b/curricula/cs336_a1/modules/cosine_schedule/validator.sh index 7dee0fc407..0626dab5d3 100755 --- a/curricula/cs336_a1/modules/cosine_schedule/validator.sh +++ b/curricula/cs336_a1/modules/cosine_schedule/validator.sh @@ -15,7 +15,7 @@ fi # HARDEN STAGE: File already copied by submit-fix, just cd to shadow worktree if [ "$(pwd)" != "$SHADOW_WORKTREE" ]; then # BUILD STAGE: We're in main directory, copy file and cd to shadow worktree - cp cs336_basics/optimizer.py "$SHADOW_WORKTREE/cs336_basics/optimizer.py" + cp cs336_basics/utils.py "$SHADOW_WORKTREE/cs336_basics/utils.py" cd "$SHADOW_WORKTREE" else # HARDEN STAGE: Already in shadow worktree, file was copied by submit-fix @@ -30,14 +30,14 @@ start_time=$(python3 -c 'import time; print(time.time())') if [ -n "$MASTERY_PYTHON" ]; then # Engine provided its Python executable - use it export PYTHONPATH="$(pwd):$PYTHONPATH" - "$MASTERY_PYTHON" -m pytest tests/test_optimizer.py::test_lr_cosine_schedule -v --tb=short --import-mode=importlib + "$MASTERY_PYTHON" -m pytest tests/test_optimizer.py::test_get_lr_cosine_schedule -v --tb=short --import-mode=importlib elif [ -n "$VIRTUAL_ENV" ]; then # We're in an active virtual environment - use its Python explicitly export PYTHONPATH="$(pwd):$PYTHONPATH" - "$VIRTUAL_ENV/bin/python" -m pytest tests/test_optimizer.py::test_lr_cosine_schedule -v --tb=short --import-mode=importlib + "$VIRTUAL_ENV/bin/python" -m pytest tests/test_optimizer.py::test_get_lr_cosine_schedule -v --tb=short --import-mode=importlib else # No active environment - use uv to create one - uv run pytest tests/test_optimizer.py::test_lr_cosine_schedule -v --tb=short --import-mode=importlib + uv run pytest tests/test_optimizer.py::test_get_lr_cosine_schedule -v --tb=short --import-mode=importlib fi # Record end time diff --git a/curricula/cs336_a1/modules/cross_entropy/bugs/no_logsumexp.json b/curricula/cs336_a1/modules/cross_entropy/bugs/no_logsumexp.json index 0c5c2f8722..6292600a9a 100644 --- a/curricula/cs336_a1/modules/cross_entropy/bugs/no_logsumexp.json +++ b/curricula/cs336_a1/modules/cross_entropy/bugs/no_logsumexp.json @@ -1,6 +1,6 @@ { "id": "cross-entropy-no-logsumexp", - "description": "Using naive softmax instead of logsumexp", + "description": "Replace log-sum-exp with a plain max, breaking the normalization term.", "injection_type": "ast", "engine_version": "2.1", "target_function": "cross_entropy", @@ -13,34 +13,14 @@ "targets": [ { "node_type": "Name", - "id": "log_sum_exp" + "id": "lse" } ] }, "replacement": { - "type": "delete_statement" - } - }, - { - "pass": 2, - "type": "find_and_replace", - "pattern": { - "node_type": "Assign", - "targets": [ - { - "node_type": "Name", - "id": "log_probs" - } - ], - "value": { - "node_type": "BinOp", - "op": "Sub" - } - }, - "replacement": { - "type": "replace_with", - "source": "exp_logits = torch.exp(x32)\nsoftmax_probs = exp_logits / exp_logits.sum(dim=-1, keepdim=True)\ntarget_probs = softmax_probs.gather(dim=-1, index=t.unsqueeze(-1)).squeeze(-1)\nlog_probs = torch.log(target_probs + 1e-10)" + "type": "replace_value_with", + "source": "x32.max(dim=-1).values" } } ] -} \ No newline at end of file +} diff --git a/curricula/cs336_a1/modules/cross_entropy/bugs/no_logsumexp.patch b/curricula/cs336_a1/modules/cross_entropy/bugs/no_logsumexp.patch deleted file mode 100644 index cfa3909f25..0000000000 --- a/curricula/cs336_a1/modules/cross_entropy/bugs/no_logsumexp.patch +++ /dev/null @@ -1,21 +0,0 @@ ---- cs336_basics/utils.py -+++ cs336_basics/utils.py -@@ -38,9 +38,12 @@ - logits = inputs - orig_dtype = logits.dtype - x32 = logits.float() - t = targets.long() -- # log-sum-exp for stability -- lse = torch.logsumexp(x32, dim=-1) -- # pick the logit for the correct class -- correct = x32.gather(dim=-1, index=t.unsqueeze(-1)).squeeze(-1) -- loss = (lse - correct).mean() -+ # BUG: Naive softmax + log instead of logsumexp - causes numerical instability! -+ # Compute softmax explicitly (without subtract-max trick for additional instability) -+ exp_logits = torch.exp(x32) -+ softmax_probs = exp_logits / exp_logits.sum(dim=-1, keepdim=True) -+ # Extract probabilities for target classes -+ target_probs = softmax_probs.gather(dim=-1, index=t.unsqueeze(-1)).squeeze(-1) -+ # Take log of probabilities (can produce -inf if prob is 0) -+ loss = -torch.log(target_probs).mean() - return loss.to(orig_dtype) diff --git a/curricula/cs336_a1/modules/cross_entropy/bugs/no_logsumexp_draft.json b/curricula/cs336_a1/modules/cross_entropy/bugs/no_logsumexp_draft.json deleted file mode 100644 index 878bf6acee..0000000000 --- a/curricula/cs336_a1/modules/cross_entropy/bugs/no_logsumexp_draft.json +++ /dev/null @@ -1,126 +0,0 @@ -{ - "id": "cross_entropy-naive-softmax", - "description": "Replaces logsumexp with naive softmax and log, causing numerical instability.", - "injection_type": "ast", - "engine_version": "2.1", - "target_function": "cross_entropy", - "logic": [ - { - "pass_": 1, - "type": "find_and_replace", - "description": "Replace logsumexp with naive softmax computation and log.", - "pattern": { - "node_type": "Assign", - "targets": [ - { - "node_type": "Name", - "id": "lse" - } - ], - "value": { - "node_type": "Call", - "op": null, - "attr": "logsumexp" - }, - "attr": null, - "op": null, - "func": { - "node_type": "Attribute", - "op": null, - "attr": "logsumexp" - }, - "left": null, - "right": null, - "args": null, - "keywords": null - }, - "conditions": null, - "track_as": null, - "replacement": { - "type": "replace_value_with", - "source": "torch.log(torch.exp(x32) / torch.exp(x32).sum(dim=-1, keepdim=True))", - "name": null - } - }, - { - "pass_": 2, - "type": "find_and_replace", - "description": "Replace correct class logit extraction with probability extraction from softmax.", - "pattern": { - "node_type": "Assign", - "targets": [ - { - "node_type": "Name", - "id": "correct" - } - ], - "value": { - "node_type": "Call", - "op": null, - "attr": "gather" - }, - "attr": null, - "op": null, - "func": { - "node_type": "Attribute", - "op": null, - "attr": "gather" - }, - "left": null, - "right": null, - "args": null, - "keywords": null - }, - "conditions": null, - "track_as": null, - "replacement": { - "type": "replace_value_with", - "source": "softmax_probs.gather(dim=-1, index=t.unsqueeze(-1)).squeeze(-1)", - "name": null - } - }, - { - "pass_": 3, - "type": "find_and_replace", - "description": "Replace loss computation using logsumexp with naive log of probabilities.", - "pattern": { - "node_type": "Assign", - "targets": [ - { - "node_type": "Name", - "id": "loss" - } - ], - "value": { - "node_type": "Call", - "op": null, - "attr": "mean" - }, - "attr": null, - "op": null, - "func": { - "node_type": "Attribute", - "op": null, - "attr": "mean" - }, - "left": null, - "right": null, - "args": null, - "keywords": null - }, - "conditions": null, - "track_as": null, - "replacement": { - "type": "replace_value_with", - "source": "-torch.log(target_probs).mean()", - "name": null - } - } - ], - "metadata": { - "created": "2025-11-13", - "version": "2.0", - "author": "LLM-Generated", - "tier": "complex" - } -} \ No newline at end of file diff --git a/curricula/cs336_a1/modules/cross_entropy/bugs/no_logsumexp_symptom.txt b/curricula/cs336_a1/modules/cross_entropy/bugs/no_logsumexp_symptom.txt index fe121f4e9c..f4438ca859 100644 --- a/curricula/cs336_a1/modules/cross_entropy/bugs/no_logsumexp_symptom.txt +++ b/curricula/cs336_a1/modules/cross_entropy/bugs/no_logsumexp_symptom.txt @@ -1,59 +1,14 @@ -# Bug Symptom: Numerical Instability in Cross-Entropy +# Bug Symptom: Cross Entropy — cross-entropy-no-logsumexp ## Observed Behavior -Your cross-entropy implementation produces `NaN` (Not a Number) or `inf` (infinity) values when processing inputs with large-magnitude logits. - -## Failing Test Case -```python -logits = torch.tensor([[100.0, 0.0, 0.0]]) # Large logit for class 0 -targets = torch.tensor([1]) # True class is 1 (not the largest logit) - -result = cross_entropy(logits, targets) -# Expected: reasonable loss value (~100.0) -# Actual: inf or nan -``` - -## Error Message -``` -AssertionError: Cross-entropy output contains NaN or Inf values -``` +A subtle semantic bug has been injected into your `cross_entropy` implementation. Your code +runs without raising a syntax error, but the module's correctness test now fails: +Replace log-sum-exp with a plain max, breaking the normalization term. ## Your Challenge -The bug arises from computing softmax explicitly before taking the log. This two-step process has two failure modes: - -1. **Overflow in `exp()`**: For large positive logits (e.g., 100.0), `exp(100) ≈ 10^43` overflows to `inf` -2. **Underflow in softmax**: Even if overflow is avoided, softmax can produce probabilities so small they underflow to exactly 0.0, and `log(0) = -inf` - -**Debug this issue by:** -1. Identifying whether the bug manifests as NaN (from inf/inf) or -inf (from log(0)) -2. Understanding why the two-step approach (softmax → log) is numerically unstable -3. Implementing the log-sum-exp formulation directly: `logsumexp(logits) - logits[target]` -4. Verifying the output matches PyTorch's `F.cross_entropy` - -## Expected Output After Fix -After your fix, `cross_entropy(torch.tensor([[100.0, 0.0, 0.0]]), torch.tensor([1]))` should produce a finite loss value around 100.0, not infinity or NaN. - -## Debugging Tips -- Print intermediate values: `exp_logits`, `softmax_probs`, `target_probs` before taking log -- Check for overflow: `torch.isinf(exp_logits).any()` -- Check for underflow: `(softmax_probs == 0.0).any()` -- Try the test case with smaller logits (e.g., `[10.0, 0.0, 0.0]`) to see if it works - -## Mathematical Insight -The log-sum-exp trick computes `log(sum(exp(logits)))` stably without materializing the intermediate `sum(exp(logits))` which could overflow. PyTorch's `logsumexp` uses the identity: - -``` -logsumexp(x) = max(x) + log(sum(exp(x - max(x)))) -``` - -This keeps all exponentials in a safe range while computing the mathematically exact result. - -## Hint -Cross-entropy can be computed directly as: -```python -loss = logsumexp(logits, dim=-1) - logits[target_index] -``` - -This never computes softmax probabilities explicitly, avoiding both overflow in exp() and underflow in the probabilities. +A single line in `cs336_basics/utils.py` no longer matches your correct implementation. Read the +function, form a hypothesis from the symptom above, and locate the divergence. +Restore the correct behavior — do not rewrite the whole function. -Run `engine submit-fix` once you've fixed the numerical stability issue. +## Verify +Run `mastery submit` to re-run the validator once you believe the bug is fixed. diff --git a/curricula/cs336_a1/modules/data_loader/bugs/wrong_sampling_range.patch b/curricula/cs336_a1/modules/data_loader/bugs/wrong_sampling_range.patch deleted file mode 100644 index e91569d277..0000000000 --- a/curricula/cs336_a1/modules/data_loader/bugs/wrong_sampling_range.patch +++ /dev/null @@ -1,15 +0,0 @@ ---- a/cs336_basics/utils.py -+++ b/cs336_basics/utils.py -@@ -10,8 +10,10 @@ def get_batch(dataset, batch_size, context_length, device): - if n <= 0: - raise ValueError("Dataset too short") - -- # Sample starting positions -- starts = torch.randint(low=0, high=n, size=(batch_size,)) -+ # BUG: Wrong sampling range! Using high=n+1 instead of high=n -+ # This can sample position n-1+1=n, which needs tokens up to n+L (out of bounds!) -+ # Should be: high=n (exclusive), which gives valid range [0, n-1] -+ starts = torch.randint(low=0, high=n+1, size=(batch_size,)) # WRONG! - - # Create indices - offsets = torch.arange(context_length).unsqueeze(0) diff --git a/curricula/cs336_a1/modules/data_loader/bugs/wrong_sampling_range_draft.json b/curricula/cs336_a1/modules/data_loader/bugs/wrong_sampling_range_draft.json deleted file mode 100644 index b7de208b4e..0000000000 --- a/curricula/cs336_a1/modules/data_loader/bugs/wrong_sampling_range_draft.json +++ /dev/null @@ -1,68 +0,0 @@ -{ - "id": "data_loader-wrong-sampling-range", - "description": "Introduces a bug by using an incorrect high value in torch.randint, causing out-of-bounds sampling.", - "injection_type": "ast", - "engine_version": "2.1", - "target_function": "data_loader", - "logic": [ - { - "pass_": 1, - "type": "find_and_replace", - "description": "Find torch.randint call with high=n and change to high=n+1", - "pattern": { - "node_type": "Assign", - "targets": [ - { - "node_type": "Name", - "id": "starts" - } - ], - "value": { - "node_type": "Call", - "op": null, - "attr": null - }, - "attr": null, - "op": null, - "func": { - "node_type": "Attribute", - "op": null, - "attr": "randint" - }, - "left": null, - "right": null, - "args": null, - "keywords": [ - { - "arg": "high", - "value": { - "node_type": "Name", - "op": null, - "attr": null - } - } - ] - }, - "conditions": [ - { - "check": "target_is_name", - "value": null, - "index": 0, - "name": null - } - ], - "track_as": null, - "replacement": { - "type": "replace_with", - "source": "node.value.keywords[0].value + 1", - "name": null - } - } - ], - "metadata": { - "created": "2025-11-13", - "version": "2.0", - "author": "LLM-Generated", - "tier": "simple" - } -} \ No newline at end of file diff --git a/curricula/cs336_a1/modules/data_loader/bugs/wrong_sampling_range_symptom.txt b/curricula/cs336_a1/modules/data_loader/bugs/wrong_sampling_range_symptom.txt new file mode 100644 index 0000000000..0ce74c99c3 --- /dev/null +++ b/curricula/cs336_a1/modules/data_loader/bugs/wrong_sampling_range_symptom.txt @@ -0,0 +1,14 @@ +# Bug Symptom: Data Loader — data-loader-wrong-sampling-range + +## Observed Behavior +A subtle semantic bug has been injected into your `get_batch` implementation. Your code +runs without raising a syntax error, but the module's correctness test now fails: +Wrong high bound in randint causing out-of-bounds + +## Your Challenge +A single line in `cs336_basics/utils.py` no longer matches your correct implementation. Read the +function, form a hypothesis from the symptom above, and locate the divergence. +Restore the correct behavior — do not rewrite the whole function. + +## Verify +Run `mastery submit` to re-run the validator once you believe the bug is fixed. diff --git a/curricula/cs336_a1/modules/data_loader/validator.sh b/curricula/cs336_a1/modules/data_loader/validator.sh index d105817acc..580d5fa633 100755 --- a/curricula/cs336_a1/modules/data_loader/validator.sh +++ b/curricula/cs336_a1/modules/data_loader/validator.sh @@ -20,12 +20,12 @@ start_time=$(python3 -c 'import time; print(time.time())') if [ -n "$MASTERY_PYTHON" ]; then export PYTHONPATH="$(pwd):$PYTHONPATH" - "$MASTERY_PYTHON" -m pytest tests/test_training.py::test_get_batch -v --tb=short --import-mode=importlib + "$MASTERY_PYTHON" -m pytest tests/test_data.py::test_get_batch -v --tb=short --import-mode=importlib elif [ -n "$VIRTUAL_ENV" ]; then export PYTHONPATH="$(pwd):$PYTHONPATH" - "$VIRTUAL_ENV/bin/python" -m pytest tests/test_training.py::test_get_batch -v --tb=short --import-mode=importlib + "$VIRTUAL_ENV/bin/python" -m pytest tests/test_data.py::test_get_batch -v --tb=short --import-mode=importlib else - uv run pytest tests/test_training.py::test_get_batch -v --tb=short --import-mode=importlib + uv run pytest tests/test_data.py::test_get_batch -v --tb=short --import-mode=importlib fi end_time=$(python3 -c 'import time; print(time.time())') diff --git a/curricula/cs336_a1/modules/embedding/bugs/wrong_dimension_order.json b/curricula/cs336_a1/modules/embedding/bugs/wrong_dimension_order.json index 0de1e26d54..9f47458dd9 100644 --- a/curricula/cs336_a1/modules/embedding/bugs/wrong_dimension_order.json +++ b/curricula/cs336_a1/modules/embedding/bugs/wrong_dimension_order.json @@ -1,6 +1,6 @@ { "id": "embedding-wrong-dimension-order", - "description": "Swapped num_embeddings and embedding_dim", + "description": "Embedding weight allocated with (embedding_dim, num_embeddings) \u2014 dims swapped.", "injection_type": "ast", "engine_version": "2.1", "target_function": "__init__", @@ -10,20 +10,42 @@ "type": "find_and_replace", "pattern": { "node_type": "Assign", - "targets": [ - { - "node_type": "Attribute", - "attr": "embedding" - } - ], "value": { - "node_type": "Call" + "node_type": "Call", + "func": { + "node_type": "Attribute", + "attr": "Parameter" + }, + "args": [ + { + "node_type": "Call", + "func": { + "node_type": "Attribute", + "attr": "empty" + }, + "args": [ + { + "node_type": "Tuple", + "elts": [ + { + "node_type": "Name", + "id": "num_embeddings" + }, + { + "node_type": "Name", + "id": "embedding_dim" + } + ] + } + ] + } + ] } }, "replacement": { - "type": "replace_with", - "source": "self.embedding = nn.Embedding(num_embeddings=embedding_dim, embedding_dim=num_embeddings)" + "type": "replace_value_with", + "source": "nn.Parameter(torch.empty((embedding_dim, num_embeddings)))" } } ] -} \ No newline at end of file +} diff --git a/curricula/cs336_a1/modules/embedding/bugs/wrong_dimension_order.patch b/curricula/cs336_a1/modules/embedding/bugs/wrong_dimension_order.patch deleted file mode 100644 index 716220c3ee..0000000000 --- a/curricula/cs336_a1/modules/embedding/bugs/wrong_dimension_order.patch +++ /dev/null @@ -1,15 +0,0 @@ ---- a/cs336_basics/layers.py -+++ b/cs336_basics/layers.py -@@ -25,8 +25,9 @@ class Embedding(nn.Module): - def __init__(self, num_embeddings, embedding_dim, device=None, dtype=None): - super().__init__() -- self.embedding = nn.Embedding( -- num_embeddings=num_embeddings, -- embedding_dim=embedding_dim, -+ # BUG: Swapped dimension order! nn.Embedding expects (num_embeddings, embedding_dim) -+ # not (embedding_dim, num_embeddings). This creates wrong-shaped embedding matrix. -+ self.embedding = nn.Embedding( -+ num_embeddings=embedding_dim, # Should be num_embeddings! -+ embedding_dim=num_embeddings, # Should be embedding_dim! - device=device, - dtype=dtype, diff --git a/curricula/cs336_a1/modules/embedding/bugs/wrong_dimension_order_draft.json b/curricula/cs336_a1/modules/embedding/bugs/wrong_dimension_order_draft.json deleted file mode 100644 index d2464ce3cb..0000000000 --- a/curricula/cs336_a1/modules/embedding/bugs/wrong_dimension_order_draft.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "id": "embedding-dimension-swap", - "description": "Swaps the order of num_embeddings and embedding_dim in nn.Embedding, causing incorrect embedding matrix shape.", - "injection_type": "ast", - "engine_version": "2.1", - "target_function": "__init__", - "logic": [ - { - "pass_": 1, - "type": "find_and_replace", - "description": "Swap num_embeddings and embedding_dim arguments in nn.Embedding call", - "pattern": { - "node_type": "Assign", - "targets": [ - { - "node_type": "Name", - "id": "self.embedding" - } - ], - "value": { - "node_type": "Call", - "op": null, - "attr": null - }, - "attr": null, - "op": null, - "func": { - "node_type": "Attribute", - "op": null, - "attr": "Embedding" - }, - "left": null, - "right": null, - "args": [ - { - "node_type": "Name", - "op": null, - "attr": null - }, - { - "node_type": "Name", - "op": null, - "attr": null - } - ], - "keywords": [ - { - "arg": "num_embeddings", - "value": { - "node_type": "Name", - "op": null, - "attr": null - } - }, - { - "arg": "embedding_dim", - "value": { - "node_type": "Name", - "op": null, - "attr": null - } - } - ] - }, - "conditions": null, - "track_as": null, - "replacement": { - "type": "replace_with", - "source": "node.value.keywords[1].value", - "name": null - } - }, - { - "pass_": 2, - "type": "find_and_replace", - "description": "Swap embedding_dim and num_embeddings arguments in nn.Embedding call", - "pattern": { - "node_type": "Assign", - "targets": [ - { - "node_type": "Name", - "id": "self.embedding" - } - ], - "value": { - "node_type": "Call", - "op": null, - "attr": null - }, - "attr": null, - "op": null, - "func": { - "node_type": "Attribute", - "op": null, - "attr": "Embedding" - }, - "left": null, - "right": null, - "args": [ - { - "node_type": "Name", - "op": null, - "attr": null - }, - { - "node_type": "Name", - "op": null, - "attr": null - } - ], - "keywords": [ - { - "arg": "embedding_dim", - "value": { - "node_type": "Name", - "op": null, - "attr": null - } - }, - { - "arg": "num_embeddings", - "value": { - "node_type": "Name", - "op": null, - "attr": null - } - } - ] - }, - "conditions": null, - "track_as": null, - "replacement": { - "type": "replace_with", - "source": "node.value.keywords[0].value", - "name": null - } - } - ], - "metadata": { - "created": "2025-11-13", - "version": "2.0", - "author": "LLM-Generated", - "tier": "simple" - } -} \ No newline at end of file diff --git a/curricula/cs336_a1/modules/embedding/bugs/wrong_dimension_order_symptom.txt b/curricula/cs336_a1/modules/embedding/bugs/wrong_dimension_order_symptom.txt new file mode 100644 index 0000000000..557e8cde38 --- /dev/null +++ b/curricula/cs336_a1/modules/embedding/bugs/wrong_dimension_order_symptom.txt @@ -0,0 +1,14 @@ +# Bug Symptom: Embedding — embedding-wrong-dimension-order + +## Observed Behavior +A subtle semantic bug has been injected into your `__init__` implementation. Your code +runs without raising a syntax error, but the module's correctness test now fails: +Embedding weight allocated with (embedding_dim, num_embeddings) — dims swapped. + +## Your Challenge +A single line in `cs336_basics/layers.py` no longer matches your correct implementation. Read the +function, form a hypothesis from the symptom above, and locate the divergence. +Restore the correct behavior — do not rewrite the whole function. + +## Verify +Run `mastery submit` to re-run the validator once you believe the bug is fixed. diff --git a/curricula/cs336_a1/modules/gradient_clipping/bugs/per_parameter_clipping.json b/curricula/cs336_a1/modules/gradient_clipping/bugs/per_parameter_clipping.json index 33cba61dbb..38f9f9171b 100644 --- a/curricula/cs336_a1/modules/gradient_clipping/bugs/per_parameter_clipping.json +++ b/curricula/cs336_a1/modules/gradient_clipping/bugs/per_parameter_clipping.json @@ -1,30 +1,26 @@ { - "id": "gradient-clipping-per-parameter", - "description": "Per-parameter clipping instead of global norm", + "id": "gradient-clipping-wrong-norm", + "description": "Global gradient norm computed with L1 (norm(1)) instead of L2.", "injection_type": "ast", "engine_version": "2.1", - "target_function": "clip_gradients_by_global_norm", + "target_function": "gradient_clipping", "logic": [ { "pass": 1, "type": "find_and_replace", "pattern": { "node_type": "Assign", - "targets": [{"node_type": "Name", "id": "total_norm"}] - }, - "replacement": {"type": "delete_statement"} - }, - { - "pass": 2, - "type": "find_and_replace", - "pattern": { - "node_type": "If", - "test": {"node_type": "Compare"} + "targets": [ + { + "node_type": "Name", + "id": "total_norm" + } + ] }, "replacement": { - "type": "replace_with", - "source": "for g in grads:\n grad_norm = g.detach().norm(2)\n if grad_norm > max_l2_norm:\n clip_coef = max_l2_norm / (grad_norm + 1e-6)\n g.mul_(clip_coef)" + "type": "replace_value_with", + "source": "norms.norm(1)" } } ] -} \ No newline at end of file +} diff --git a/curricula/cs336_a1/modules/gradient_clipping/bugs/per_parameter_clipping.patch b/curricula/cs336_a1/modules/gradient_clipping/bugs/per_parameter_clipping.patch deleted file mode 100644 index 04bd769529..0000000000 --- a/curricula/cs336_a1/modules/gradient_clipping/bugs/per_parameter_clipping.patch +++ /dev/null @@ -1,23 +0,0 @@ ---- cs336_basics/utils.py -+++ cs336_basics/utils.py -@@ -60,13 +60,11 @@ - """ - # Collect grads that exist - grads = [p.grad for p in parameters if getattr(p, "grad", None) is not None] - if not grads: - return -- # Compute global L2 norm (same as norm of concatenation) -- norms = torch.stack([g.detach().norm(2) for g in grads]) -- total_norm = norms.norm(2) -- # Only scale if norm exceeds the threshold -- if total_norm > max_l2_norm: -- clip_coef = max_l2_norm / (total_norm + 1e-6) -- for g in grads: -- g.mul_(clip_coef) -+ # BUG: Per-parameter clipping instead of global norm - distorts gradient direction! -+ for g in grads: -+ grad_norm = g.detach().norm(2) -+ if grad_norm > max_l2_norm: -+ # Clip each gradient independently -+ clip_coef = max_l2_norm / (grad_norm + 1e-6) -+ g.mul_(clip_coef) diff --git a/curricula/cs336_a1/modules/gradient_clipping/bugs/per_parameter_clipping_draft.json b/curricula/cs336_a1/modules/gradient_clipping/bugs/per_parameter_clipping_draft.json deleted file mode 100644 index 83a3c025e0..0000000000 --- a/curricula/cs336_a1/modules/gradient_clipping/bugs/per_parameter_clipping_draft.json +++ /dev/null @@ -1,83 +0,0 @@ -{ - "id": "gradient_clipping-per-parameter-bug", - "description": "Changes global gradient clipping to per-parameter clipping, distorting gradient direction.", - "injection_type": "ast", - "engine_version": "2.1", - "target_function": "gradient_clipping", - "logic": [ - { - "pass_": 1, - "type": "find_and_replace", - "description": "Replace global L2 norm computation and scaling with per-parameter clipping logic.", - "pattern": { - "node_type": "Assign", - "targets": [ - { - "node_type": "Name", - "id": "total_norm" - } - ], - "value": { - "node_type": "Call", - "op": null, - "attr": "norm" - }, - "attr": null, - "op": null, - "func": { - "node_type": "Attribute", - "op": null, - "attr": "norm" - }, - "left": null, - "right": null, - "args": null, - "keywords": null - }, - "conditions": null, - "track_as": null, - "replacement": { - "type": "replace_with", - "source": "node.value.func.value.args[0]", - "name": null - } - }, - { - "pass_": 2, - "type": "find_and_replace", - "description": "Replace scaling logic with per-parameter clipping logic.", - "pattern": { - "node_type": "If", - "targets": null, - "value": null, - "attr": null, - "op": null, - "func": null, - "left": null, - "right": null, - "args": null, - "keywords": null - }, - "conditions": [ - { - "check": "has_keyword_arg", - "value": null, - "index": null, - "name": "total_norm" - } - ], - "track_as": null, - "replacement": { - "type": "replace_with", - "source": "node.body[0]", - "name": null - } - } - ], - "metadata": { - "created": "2025-11-13", - "version": "2.0", - "author": "LLM-Generated", - "tier": "complex" - } -} \ No newline at end of file diff --git a/curricula/cs336_a1/modules/gradient_clipping/bugs/per_parameter_clipping_symptom.txt b/curricula/cs336_a1/modules/gradient_clipping/bugs/per_parameter_clipping_symptom.txt index 07871bb566..05e152da86 100644 --- a/curricula/cs336_a1/modules/gradient_clipping/bugs/per_parameter_clipping_symptom.txt +++ b/curricula/cs336_a1/modules/gradient_clipping/bugs/per_parameter_clipping_symptom.txt @@ -1,71 +1,14 @@ -# Bug Symptom: Incorrect Gradient Direction After Clipping +# Bug Symptom: Gradient Clipping — gradient-clipping-wrong-norm ## Observed Behavior -Your gradient clipping implementation produces different results than `torch.nn.utils.clip_grad_norm_` even though individual gradient norms are within the threshold. The test detects that relative magnitudes between parameters are being distorted. - -## Failing Test Case -```python -# Two parameters with different gradient magnitudes -param1 = torch.nn.Parameter(torch.zeros(10)) -param2 = torch.nn.Parameter(torch.zeros(10)) - -param1.grad = torch.ones(10) * 8.0 # norm = 8.0 -param2.grad = torch.ones(10) * 2.0 # norm = 2.0 - -# Global norm = sqrt(64 + 4) ≈ 8.25 -# Expected behavior with max_norm=5.0: -# Scale factor = 5.0 / 8.25 ≈ 0.606 -# param1.grad becomes ones(10) * 4.85 # 8.0 * 0.606 -# param2.grad becomes ones(10) * 1.21 # 2.0 * 0.606 -# Ratio preserved: 4.85 / 1.21 ≈ 4.0 (same as original 8.0 / 2.0) - -gradient_clipping([param1, param2], max_l2_norm=5.0) - -# Actual (buggy) behavior: -# param1.grad becomes ones(10) * 5.0 # clipped to max_norm -# param2.grad stays ones(10) * 2.0 # not clipped (below threshold) -# Ratio distorted: 5.0 / 2.0 = 2.5 (not 4.0!) -``` - -## Error Message -``` -AssertionError: Gradient clipping output doesn't match torch.nn.utils.clip_grad_norm_ -``` +A subtle semantic bug has been injected into your `gradient_clipping` implementation. Your code +runs without raising a syntax error, but the module's correctness test now fails: +Global gradient norm computed with L1 (norm(1)) instead of L2. ## Your Challenge -The bug is that you're clipping each parameter's gradients independently instead of computing a **global norm** and scaling **all gradients proportionally**. This destroys the relative magnitudes between parameters, effectively changing the direction of the optimization update. - -**Debug this issue by:** -1. Understanding why per-parameter clipping is fundamentally incorrect -2. Implementing the global norm computation: `sqrt(sum of all gradient norms squared)` -3. Computing a single scaling factor based on the global norm -4. Applying that scaling factor to ALL gradients, not just those exceeding the threshold - -## Expected Behavior After Fix -After your fix, when the global norm exceeds the threshold: -- ALL gradients should be scaled by the same factor: `max_norm / global_norm` -- This preserves the 4:1 ratio in the example above -- The direction of the combined gradient update is maintained -- Only the magnitude is reduced to the threshold - -## Debugging Tips -- Print gradient norms before and after clipping for each parameter -- Compute the global norm manually: `sqrt(norm1^2 + norm2^2 + ...)` -- Check if the ratio of gradient magnitudes is preserved after clipping -- Compare with the ratio that `torch.nn.utils.clip_grad_norm_` produces - -## Mathematical Insight -Gradient descent updates parameters by: `param -= learning_rate * grad` - -The combined update direction is determined by the **relative magnitudes** of all gradients. Per-parameter clipping changes these ratios, pointing the optimization in a different direction than intended. Global clipping maintains the direction while only limiting the step size. - -## Hint -The correct algorithm is: -1. Collect all gradients -2. Compute global_norm = `torch.stack([g.norm(2) for g in grads]).norm(2)` -3. If global_norm > max_norm: compute `clip_coef = max_norm / global_norm` -4. Scale **all** gradients by clip_coef (even those individually below threshold) - -This ensures proportional scaling that preserves the optimization direction. +A single line in `cs336_basics/utils.py` no longer matches your correct implementation. Read the +function, form a hypothesis from the symptom above, and locate the divergence. +Restore the correct behavior — do not rewrite the whole function. -Run `engine submit-fix` once you've fixed the clipping algorithm. +## Verify +Run `mastery submit` to re-run the validator once you believe the bug is fixed. diff --git a/curricula/cs336_a1/modules/linear/bugs/missing_transpose.patch b/curricula/cs336_a1/modules/linear/bugs/missing_transpose.patch deleted file mode 100644 index b385f96fa4..0000000000 --- a/curricula/cs336_a1/modules/linear/bugs/missing_transpose.patch +++ /dev/null @@ -1,14 +0,0 @@ ---- a/cs336_basics/layers.py -+++ b/cs336_basics/layers.py -@@ -35,8 +35,9 @@ class Linear(nn.Module): - return y * self.weight - - def forward(self, in_features: Float[Tensor, " ... in_features"]) -> Float[Tensor, " ... out_features"]: -- # Compute y = x @ W^T -- y = in_features.matmul(self.weight.t()) -+ # BUG: Missing transpose! This causes dimension mismatch since weight is (out, in) -+ # Should be: y = in_features.matmul(self.weight.t()) -+ y = in_features.matmul(self.weight) # Wrong! Dimensions don't match - - if self.bias is not None: - y = y + self.bias diff --git a/curricula/cs336_a1/modules/linear/bugs/missing_transpose_draft.json b/curricula/cs336_a1/modules/linear/bugs/missing_transpose_draft.json deleted file mode 100644 index 5e9022709d..0000000000 --- a/curricula/cs336_a1/modules/linear/bugs/missing_transpose_draft.json +++ /dev/null @@ -1,63 +0,0 @@ -{ - "id": "linear-missing-transpose", - "description": "Bug in linear: Missing transpose on weight matrix causing dimension mismatch.", - "injection_type": "ast", - "engine_version": "2.1", - "target_function": "forward", - "logic": [ - { - "pass_": 1, - "type": "find_and_replace", - "description": "Find in_features.matmul(self.weight) and replace with in_features.matmul(self.weight.t())", - "pattern": { - "node_type": "Assign", - "targets": [ - { - "node_type": "Name", - "id": "y" - } - ], - "value": { - "node_type": "Call", - "op": null, - "attr": null - }, - "attr": null, - "op": null, - "func": { - "node_type": "Attribute", - "op": null, - "attr": "matmul" - }, - "left": null, - "right": null, - "args": [ - { - "node_type": "Name", - "op": null, - "attr": null - }, - { - "node_type": "Attribute", - "op": null, - "attr": "weight" - } - ], - "keywords": null - }, - "conditions": null, - "track_as": null, - "replacement": { - "type": "replace_value_with", - "source": "y.matmul(self.weight.t())", - "name": null - } - } - ], - "metadata": { - "created": "2025-11-13", - "version": "2.0", - "author": "LLM-Generated", - "tier": "simple" - } -} \ No newline at end of file diff --git a/curricula/cs336_a1/modules/linear/bugs/missing_transpose_draft_v2.json b/curricula/cs336_a1/modules/linear/bugs/missing_transpose_draft_v2.json deleted file mode 100644 index 7de46c45b3..0000000000 --- a/curricula/cs336_a1/modules/linear/bugs/missing_transpose_draft_v2.json +++ /dev/null @@ -1,35 +0,0 @@ -{ - "id": "linear-missing-transpose", - "description": "Missing transpose on weight matrix in linear layer", - "injection_type": "ast", - "engine_version": "2.1", - "target_function": "forward", - "logic": [ - { - "pass": 1, - "type": "find_and_replace", - "description": "Replace in_features.matmul(self.weight.t()) with in_features.matmul(self.weight)", - "pattern": { - "node_type": "Assign", - "targets": [ - { - "node_type": "Name", - "id": "y" - } - ], - "value": { - "node_type": "Call" - } - }, - "replacement": { - "type": "replace_value_with", - "source": "in_features.matmul(self.weight)" - } - } - ], - "metadata": { - "created": "2025-11-14", - "version": "2.1", - "author": "auto_fixed" - } -} \ No newline at end of file diff --git a/curricula/cs336_a1/modules/linear/bugs/missing_transpose_symptom.txt b/curricula/cs336_a1/modules/linear/bugs/missing_transpose_symptom.txt new file mode 100644 index 0000000000..67d475fbd8 --- /dev/null +++ b/curricula/cs336_a1/modules/linear/bugs/missing_transpose_symptom.txt @@ -0,0 +1,14 @@ +# Bug Symptom: Linear — linear-missing-transpose + +## Observed Behavior +A subtle semantic bug has been injected into your `forward` implementation. Your code +runs without raising a syntax error, but the module's correctness test now fails: +Missing transpose on weight matrix + +## Your Challenge +A single line in `cs336_basics/layers.py` no longer matches your correct implementation. Read the +function, form a hypothesis from the symptom above, and locate the divergence. +Restore the correct behavior — do not rewrite the whole function. + +## Verify +Run `mastery submit` to re-run the validator once you believe the bug is fixed. diff --git a/curricula/cs336_a1/modules/multihead_attention/bugs/missing_transpose_back.json b/curricula/cs336_a1/modules/multihead_attention/bugs/missing_transpose_back.json index a9f809272c..6da8ff43eb 100644 --- a/curricula/cs336_a1/modules/multihead_attention/bugs/missing_transpose_back.json +++ b/curricula/cs336_a1/modules/multihead_attention/bugs/missing_transpose_back.json @@ -1,26 +1,33 @@ { - "id": "multihead-attention-missing-transpose", - "description": "Missing transpose before concatenating heads", + "id": "multihead-attention-missing-transpose-back", + "description": "Head merge skips the transpose back to (batch, seq, heads, dim), so head and position axes are interleaved incorrectly before the output projection.", "injection_type": "ast", "engine_version": "2.1", - "target_function": "forward", + "target_function": "multihead_self_attention_with_rope", "logic": [ { "pass": 1, "type": "find_and_replace", "pattern": { - "node_type": "Expr", + "node_type": "Assign", + "targets": [ + { + "node_type": "Name", + "id": "context" + } + ], "value": { "node_type": "Call", "func": { "node_type": "Attribute", - "attr": "transpose" + "attr": "view" } } }, "replacement": { - "type": "delete_statement" + "type": "replace_value_with", + "source": "context.contiguous().view(context.shape[0], seq_len, d_model)" } } ] -} \ No newline at end of file +} diff --git a/curricula/cs336_a1/modules/multihead_attention/bugs/missing_transpose_back.patch b/curricula/cs336_a1/modules/multihead_attention/bugs/missing_transpose_back.patch deleted file mode 100644 index 66680bef5c..0000000000 --- a/curricula/cs336_a1/modules/multihead_attention/bugs/missing_transpose_back.patch +++ /dev/null @@ -1,17 +0,0 @@ ---- a/cs336_basics/layers.py -+++ b/cs336_basics/layers.py -@@ -285,10 +285,11 @@ def multihead_self_attention_with_rope( - # All heads computed in parallel - attn_output = scaled_dot_product_attention(Q_rope, K_rope, V, mask=None) - -- # Transpose back: (..., num_heads, seq_len, d_head) → (..., seq_len, num_heads, d_head) -- attn_output = attn_output.transpose(-3, -2) -+ # BUG: Missing transpose back before reshaping! -+ # Without this, we concatenate heads in wrong order, mixing information incorrectly. -+ # Should be: attn_output = attn_output.transpose(-3, -2) - -- # Concatenate heads: (..., seq_len, num_heads, d_head) → (..., seq_len, d_model) -+ # Concatenate heads: Wrong shape for concat! - attn_output = attn_output.reshape(*batch_shape, seq_len, d_model) - - # Output projection diff --git a/curricula/cs336_a1/modules/multihead_attention/bugs/missing_transpose_back_draft.json b/curricula/cs336_a1/modules/multihead_attention/bugs/missing_transpose_back_draft.json deleted file mode 100644 index 61cc225e78..0000000000 --- a/curricula/cs336_a1/modules/multihead_attention/bugs/missing_transpose_back_draft.json +++ /dev/null @@ -1,52 +0,0 @@ -{ - "id": "multihead_attention-missing-transpose", - "description": "Removes the transpose operation before reshaping, causing incorrect concatenation of attention heads.", - "injection_type": "ast", - "engine_version": "2.1", - "target_function": "multihead_attention", - "logic": [ - { - "pass_": 1, - "type": "find_and_replace", - "description": "Delete the transpose operation on attn_output before reshaping.", - "pattern": { - "node_type": "Assign", - "targets": [ - { - "node_type": "Name", - "id": "attn_output" - } - ], - "value": { - "node_type": "Call", - "op": null, - "attr": null - }, - "attr": null, - "op": null, - "func": { - "node_type": "Attribute", - "op": null, - "attr": "transpose" - }, - "left": null, - "right": null, - "args": null, - "keywords": null - }, - "conditions": null, - "track_as": null, - "replacement": { - "type": "delete_statement", - "source": null, - "name": null - } - } - ], - "metadata": { - "created": "2025-11-13", - "version": "2.0", - "author": "LLM-Generated", - "tier": "simple" - } -} \ No newline at end of file diff --git a/curricula/cs336_a1/modules/multihead_attention/bugs/missing_transpose_back_symptom.txt b/curricula/cs336_a1/modules/multihead_attention/bugs/missing_transpose_back_symptom.txt new file mode 100644 index 0000000000..f1952bed33 --- /dev/null +++ b/curricula/cs336_a1/modules/multihead_attention/bugs/missing_transpose_back_symptom.txt @@ -0,0 +1,14 @@ +# Bug Symptom: Multihead Attention — multihead-attention-missing-transpose-back + +## Observed Behavior +A subtle semantic bug has been injected into your `multihead_self_attention_with_rope` implementation. Your code +runs without raising a syntax error, but the module's correctness test now fails: +The head merge skips the transpose back to (batch, seq, heads, dim), so head and position axes are interleaved incorrectly before the output projection. + +## Your Challenge +A single line in `cs336_basics/layers.py` no longer matches your correct implementation. Read the +function, form a hypothesis from the symptom above, and locate the divergence. +Restore the correct behavior — do not rewrite the whole function. + +## Verify +Run `mastery submit` to re-run the validator once you believe the bug is fixed. diff --git a/curricula/cs336_a1/modules/rmsnorm/bugs/missing_keepdim.json b/curricula/cs336_a1/modules/rmsnorm/bugs/missing_keepdim.json index a01218fa00..fe1f7c1e41 100644 --- a/curricula/cs336_a1/modules/rmsnorm/bugs/missing_keepdim.json +++ b/curricula/cs336_a1/modules/rmsnorm/bugs/missing_keepdim.json @@ -1,14 +1,13 @@ { "id": "rmsnorm-missing-keepdim", - "description": "Removes keepdim=True from mean() call, causing incorrect broadcasting in normalization.", + "description": "mean() over the last dim without keepdim=True, breaking the broadcast.", "injection_type": "ast", "engine_version": "2.1", - "target_function": "rmsnorm", + "target_function": "forward", "logic": [ { "pass": 1, "type": "find_and_replace", - "description": "Find .mean() call with keepdim keyword and remove the keepdim argument", "pattern": { "node_type": "Call", "func": { @@ -16,23 +15,10 @@ "attr": "mean" } }, - "conditions": [ - { - "check": "has_keyword_arg", - "name": "keepdim" - } - ], "replacement": { "type": "remove_keyword_arg", "name": "keepdim" } } - ], - "metadata": { - "created": "2025-11-13", - "version": "2.0", - "author": "Cascade AI", - "tier": "medium", - "note": "Tests function argument manipulation" - } + ] } diff --git a/curricula/cs336_a1/modules/rmsnorm/bugs/missing_keepdim.patch b/curricula/cs336_a1/modules/rmsnorm/bugs/missing_keepdim.patch deleted file mode 100644 index 241f33a5a9..0000000000 --- a/curricula/cs336_a1/modules/rmsnorm/bugs/missing_keepdim.patch +++ /dev/null @@ -1,12 +0,0 @@ ---- a/cs336_basics/layers.py -+++ b/cs336_basics/layers.py -@@ -90,7 +90,8 @@ class RMSNorm(nn.Module): - orig_dtype = x.dtype - x = x.float() - # Compute RMS -- rms = torch.sqrt(torch.mean(x ** 2, dim=-1, keepdim=True)) -+ # BUG: Missing keepdim=True causes broadcasting error! -+ rms = torch.sqrt(torch.mean(x ** 2, dim=-1)) - # Normalize and scale - normalized = x / (rms + self.eps) - # Apply learned scale and convert back to original dtype diff --git a/curricula/cs336_a1/modules/rmsnorm/bugs/missing_keepdim_symptom.txt b/curricula/cs336_a1/modules/rmsnorm/bugs/missing_keepdim_symptom.txt new file mode 100644 index 0000000000..943ebfd688 --- /dev/null +++ b/curricula/cs336_a1/modules/rmsnorm/bugs/missing_keepdim_symptom.txt @@ -0,0 +1,14 @@ +# Bug Symptom: Rmsnorm — rmsnorm-missing-keepdim + +## Observed Behavior +A subtle semantic bug has been injected into your `forward` implementation. Your code +runs without raising a syntax error, but the module's correctness test now fails: +mean() over the last dim without keepdim=True, breaking the broadcast. + +## Your Challenge +A single line in `cs336_basics/layers.py` no longer matches your correct implementation. Read the +function, form a hypothesis from the symptom above, and locate the divergence. +Restore the correct behavior — do not rewrite the whole function. + +## Verify +Run `mastery submit` to re-run the validator once you believe the bug is fixed. diff --git a/curricula/cs336_a1/modules/rope/bugs/wrong_rotation.json b/curricula/cs336_a1/modules/rope/bugs/wrong_rotation.json index f21527db55..b4519970bd 100644 --- a/curricula/cs336_a1/modules/rope/bugs/wrong_rotation.json +++ b/curricula/cs336_a1/modules/rope/bugs/wrong_rotation.json @@ -1,9 +1,9 @@ { "id": "rope-wrong-rotation", - "description": "Wrong rotation formula (missing negative sign)", + "description": "Sign flip in the even rotation component (cos+sin instead of cos-sin).", "injection_type": "ast", "engine_version": "2.1", - "target_function": "apply_rotary_position_embeddings", + "target_function": "rope", "logic": [ { "pass": 1, @@ -13,18 +13,14 @@ "targets": [ { "node_type": "Name", - "id": "x_rotated_even" + "id": "x_rot_even" } - ], - "value": { - "node_type": "BinOp", - "op": "Sub" - } + ] }, "replacement": { "type": "replace_value_with", - "source": "x_even * cos_selected + x_odd * sin_selected" + "source": "x_even * cos + x_odd * sin" } } ] -} \ No newline at end of file +} diff --git a/curricula/cs336_a1/modules/rope/bugs/wrong_rotation.patch b/curricula/cs336_a1/modules/rope/bugs/wrong_rotation.patch deleted file mode 100644 index b7f705b765..0000000000 --- a/curricula/cs336_a1/modules/rope/bugs/wrong_rotation.patch +++ /dev/null @@ -1,14 +0,0 @@ ---- a/cs336_basics/layers.py -+++ b/cs336_basics/layers.py -@@ -245,8 +245,9 @@ def rope( - x_odd = x_pairs[..., 1] - - # Apply 2D rotation -- x_rotated_even = x_even * cos_selected - x_odd * sin_selected -- x_rotated_odd = x_even * sin_selected + x_odd * cos_selected -+ # BUG: Wrong rotation formula! Missing the negative sign in first equation. -+ # This breaks the rotation matrix and doesn't properly encode position. -+ x_rotated_even = x_even * cos_selected + x_odd * sin_selected # Should be minus! -+ x_rotated_odd = x_even * sin_selected + x_odd * cos_selected - - # Stack and reshape back diff --git a/curricula/cs336_a1/modules/rope/bugs/wrong_rotation_draft.json b/curricula/cs336_a1/modules/rope/bugs/wrong_rotation_draft.json deleted file mode 100644 index 3f48a1b7c9..0000000000 --- a/curricula/cs336_a1/modules/rope/bugs/wrong_rotation_draft.json +++ /dev/null @@ -1,56 +0,0 @@ -{ - "id": "rope-wrong-rotation-formula", - "description": "Introduces a bug in the 2D rotation formula by missing the negative sign, which breaks the rotation matrix.", - "injection_type": "ast", - "engine_version": "2.1", - "target_function": "apply_2d_rotation", - "logic": [ - { - "pass_": 1, - "type": "find_and_replace", - "description": "Find the correct rotation formula and replace with the buggy version missing the negative sign.", - "pattern": { - "node_type": "Assign", - "targets": [ - { - "node_type": "Name", - "id": "x_rotated_even" - } - ], - "value": { - "node_type": "BinOp", - "op": "Sub", - "attr": null - }, - "attr": null, - "op": null, - "func": null, - "left": { - "node_type": "BinOp", - "op": "Mult", - "attr": null - }, - "right": { - "node_type": "BinOp", - "op": "Mult", - "attr": null - }, - "args": null, - "keywords": null - }, - "conditions": null, - "track_as": null, - "replacement": { - "type": "replace_value_with", - "source": "node.value.left.left.id + ' * ' + node.value.left.right.id + ' + ' + node.value.right.left.id + ' * ' + node.value.right.right.id", - "name": null - } - } - ], - "metadata": { - "created": "2025-11-13", - "version": "2.0", - "author": "LLM-Generated", - "tier": "simple" - } -} \ No newline at end of file diff --git a/curricula/cs336_a1/modules/rope/bugs/wrong_rotation_symptom.txt b/curricula/cs336_a1/modules/rope/bugs/wrong_rotation_symptom.txt new file mode 100644 index 0000000000..f0994b7fdb --- /dev/null +++ b/curricula/cs336_a1/modules/rope/bugs/wrong_rotation_symptom.txt @@ -0,0 +1,14 @@ +# Bug Symptom: Rope — rope-wrong-rotation + +## Observed Behavior +A subtle semantic bug has been injected into your `rope` implementation. Your code +runs without raising a syntax error, but the module's correctness test now fails: +Sign flip in the even rotation component (cos+sin instead of cos-sin). + +## Your Challenge +A single line in `cs336_basics/layers.py` no longer matches your correct implementation. Read the +function, form a hypothesis from the symptom above, and locate the divergence. +Restore the correct behavior — do not rewrite the whole function. + +## Verify +Run `mastery submit` to re-run the validator once you believe the bug is fixed. diff --git a/curricula/cs336_a1/modules/silu/bugs/missing_multiply.patch b/curricula/cs336_a1/modules/silu/bugs/missing_multiply.patch deleted file mode 100644 index 8720c7ee5c..0000000000 --- a/curricula/cs336_a1/modules/silu/bugs/missing_multiply.patch +++ /dev/null @@ -1,12 +0,0 @@ ---- a/cs336_basics/layers.py -+++ b/cs336_basics/layers.py -@@ -73,7 +73,8 @@ def silu(in_features: Tensor) -> Tensor: - Returns: - Tensor of the same shape as input with SiLU applied element-wise - """ -- return in_features * torch.sigmoid(in_features) -+ # BUG: Missing multiplication by input! This returns just sigmoid, not SiLU. -+ return torch.sigmoid(in_features) - - - class RMSNorm(nn.Module): diff --git a/curricula/cs336_a1/modules/silu/bugs/missing_multiply_symptom.txt b/curricula/cs336_a1/modules/silu/bugs/missing_multiply_symptom.txt new file mode 100644 index 0000000000..2d8fecb6af --- /dev/null +++ b/curricula/cs336_a1/modules/silu/bugs/missing_multiply_symptom.txt @@ -0,0 +1,14 @@ +# Bug Symptom: Silu — silu-missing-multiply + +## Observed Behavior +A subtle semantic bug has been injected into your `silu` implementation. Your code +runs without raising a syntax error, but the module's correctness test now fails: +Removes the multiplication by input, returning only sigmoid(x) instead of x * sigmoid(x). + +## Your Challenge +A single line in `cs336_basics/layers.py` no longer matches your correct implementation. Read the +function, form a hypothesis from the symptom above, and locate the divergence. +Restore the correct behavior — do not rewrite the whole function. + +## Verify +Run `mastery submit` to re-run the validator once you believe the bug is fixed. diff --git a/curricula/cs336_a1/modules/softmax/bugs/no_subtract_max.patch b/curricula/cs336_a1/modules/softmax/bugs/no_subtract_max.patch deleted file mode 100644 index 2f707da837..0000000000 --- a/curricula/cs336_a1/modules/softmax/bugs/no_subtract_max.patch +++ /dev/null @@ -1,14 +0,0 @@ ---- cs336_basics/utils.py -+++ cs336_basics/utils.py -@@ -14,9 +14,8 @@ - x = in_features - orig_dtype = x.dtype - x32 = x.float() -- max_vals = x32.max(dim=dim, keepdim=True).values -- shifted = x32 - max_vals -- exps = torch.exp(shifted) -+ # BUG: Removed subtract-max trick - causes overflow! -+ exps = torch.exp(x32) - sums = exps.sum(dim=dim, keepdim=True) - out = exps / sums - return out.to(orig_dtype) diff --git a/curricula/cs336_a1/modules/softmax/bugs/no_subtract_max_v2.json b/curricula/cs336_a1/modules/softmax/bugs/no_subtract_max_v2.json deleted file mode 100644 index c3bd92f972..0000000000 --- a/curricula/cs336_a1/modules/softmax/bugs/no_subtract_max_v2.json +++ /dev/null @@ -1,72 +0,0 @@ -{ - "id": "softmax-no-subtract-max", - "description": "Removes the subtract-max trick, causing numerical overflow in softmax.", - "injection_type": "ast", - "engine_version": "2.1", - "target_function": "softmax", - "logic": [ - { - "pass": 1, - "type": "find_and_track", - "description": "Find the variable assigned the result of .max() call", - "pattern": { - "node_type": "Assign", - "value": { - "node_type": "Call", - "func": { - "node_type": "Attribute", - "attr": "max" - } - } - }, - "conditions": [ - { - "check": "targets_length_equals", - "value": 1 - }, - { - "check": "target_is_name", - "index": 0 - } - ], - "track_as": { - "max_var_name": "node.targets[0].id", - "tensor_var_name": "node.value.func.value.id" - } - }, - { - "pass": 2, - "type": "find_and_replace", - "description": "Find where tensor is subtracted by max_var and remove subtraction", - "pattern": { - "node_type": "Assign", - "value": { - "node_type": "BinOp", - "op": "Sub", - "left": { - "node_type": "Name", - "id": { - "from_context": "tensor_var_name" - } - }, - "right": { - "node_type": "Name", - "id": { - "from_context": "max_var_name" - } - } - } - }, - "replacement": { - "type": "replace_value_with", - "source": "node.value.left" - } - } - ], - "metadata": { - "created": "2025-11-13", - "version": "2.0", - "author": "Cascade AI", - "note": "Generic JSON format for Phase 3 generalization" - } -} diff --git a/curricula/cs336_a1/modules/softmax/bugs/no_subtract_max_v2_symptom.txt b/curricula/cs336_a1/modules/softmax/bugs/no_subtract_max_v2_symptom.txt deleted file mode 100644 index 7472fbd8f3..0000000000 --- a/curricula/cs336_a1/modules/softmax/bugs/no_subtract_max_v2_symptom.txt +++ /dev/null @@ -1,42 +0,0 @@ -# Bug Symptom: Numerical Overflow in Softmax - -## Observed Behavior -Your softmax implementation produces `NaN` (Not a Number) values when processing inputs with large positive values. - -## Failing Test Case -```python -x = torch.tensor([[1.0, 2.0, 3.0]]) -x_shifted = x + 100 # Shift to large positive values - -result = softmax(x_shifted, dim=1) -# Expected: valid probability distribution [0.09, 0.24, 0.67] -# Actual: tensor([[nan, nan, nan]]) -``` - -## Error Message -``` -AssertionError: Softmax output contains NaN or Inf values -``` - -## Your Challenge -The bug causes `exp()` to overflow when exponentiating large numbers (e.g., `exp(103) ≈ 10^44`), producing `inf`. When you divide `inf / inf`, the result is `NaN`. - -**Debug this issue by:** -1. Identifying which operation produces `inf` values -2. Understanding why your current implementation is numerically unstable -3. Implementing the subtract-max trick to shift inputs to a safe range -4. Verifying that the mathematical result remains unchanged - -## Expected Output After Fix -After your fix, `softmax(x + 100, dim=1)` should produce the same probability distribution as `softmax(x, dim=1)` because softmax is invariant to constant shifts. - -## Debugging Tips -- Print intermediate values: `exps`, `sums` before the division -- Check for `inf` values: `torch.isinf(exps).any()` -- Remember: `softmax(x) = softmax(x - c)` for any constant `c` -- What constant `c` would shift your largest input to 0? - -## Hint -The subtract-max trick is not an approximation—it's mathematically exact. It exploits the fact that `exp(a-c) / sum(exp(x-c)) = exp(a) / sum(exp(x))` because `exp(c)` cancels out. - -Run `engine submit-fix` once you've restored numerical stability. diff --git a/curricula/cs336_a1/modules/swiglu/bugs/missing_gate.json b/curricula/cs336_a1/modules/swiglu/bugs/missing_gate.json index df697fb0bb..a7e52da6f9 100644 --- a/curricula/cs336_a1/modules/swiglu/bugs/missing_gate.json +++ b/curricula/cs336_a1/modules/swiglu/bugs/missing_gate.json @@ -1,6 +1,6 @@ { "id": "swiglu-missing-gate", - "description": "Missing gate computation in SwiGLU", + "description": "Gate branch reuses w1 instead of w3, so the SwiGLU gating is wrong.", "injection_type": "ast", "engine_version": "2.1", "target_function": "forward", @@ -13,34 +13,21 @@ "targets": [ { "node_type": "Name", - "id": "gate" - } - ] - }, - "replacement": { - "type": "delete_statement" - } - }, - { - "pass": 2, - "type": "find_and_replace", - "pattern": { - "node_type": "Assign", - "targets": [ - { - "node_type": "Name", - "id": "gated" + "id": "b" } ], "value": { - "node_type": "BinOp", - "op": "Mult" + "node_type": "Call", + "func": { + "node_type": "Attribute", + "attr": "w3" + } } }, "replacement": { "type": "replace_value_with", - "source": "value" + "source": "self.w1(in_features)" } } ] -} \ No newline at end of file +} diff --git a/curricula/cs336_a1/modules/swiglu/bugs/missing_gate.patch b/curricula/cs336_a1/modules/swiglu/bugs/missing_gate.patch deleted file mode 100644 index e43f910d08..0000000000 --- a/curricula/cs336_a1/modules/swiglu/bugs/missing_gate.patch +++ /dev/null @@ -1,15 +0,0 @@ ---- a/cs336_basics/layers.py -+++ b/cs336_basics/layers.py -@@ -130,8 +130,9 @@ class SwiGLU(nn.Module): - # Compute value path with activation - value = silu(self.w1(x)) - -- # Compute gate path and combine -- gate = self.w3(x) -- gated = value * gate -+ # BUG: Missing gate computation! This returns just the value path -+ # without gating, reducing SwiGLU to a regular FFN with SiLU. -+ # Should be: gated = value * self.w3(x) -+ gated = value - - # Project back to d_model diff --git a/curricula/cs336_a1/modules/swiglu/bugs/missing_gate_draft.json b/curricula/cs336_a1/modules/swiglu/bugs/missing_gate_draft.json deleted file mode 100644 index b1bda2bddd..0000000000 --- a/curricula/cs336_a1/modules/swiglu/bugs/missing_gate_draft.json +++ /dev/null @@ -1,56 +0,0 @@ -{ - "id": "swiglu-missing-gate", - "description": "Removes the gate computation, reducing SwiGLU to a regular FFN with SiLU.", - "injection_type": "ast", - "engine_version": "2.1", - "target_function": "swiglu", - "logic": [ - { - "pass_": 1, - "type": "find_and_replace", - "description": "Find the gated computation and replace it with just the value path.", - "pattern": { - "node_type": "Assign", - "targets": [ - { - "node_type": "Name", - "id": "gated" - } - ], - "value": { - "node_type": "BinOp", - "op": "Mult", - "attr": null - }, - "attr": null, - "op": null, - "func": null, - "left": { - "node_type": "Name", - "op": null, - "attr": null - }, - "right": { - "node_type": "Call", - "op": null, - "attr": null - }, - "args": null, - "keywords": null - }, - "conditions": null, - "track_as": null, - "replacement": { - "type": "replace_value_with", - "source": "node.value.left", - "name": null - } - } - ], - "metadata": { - "created": "2025-11-13", - "version": "2.0", - "author": "LLM-Generated", - "tier": "simple" - } -} \ No newline at end of file diff --git a/curricula/cs336_a1/modules/swiglu/bugs/missing_gate_draft_v2.json b/curricula/cs336_a1/modules/swiglu/bugs/missing_gate_draft_v2.json deleted file mode 100644 index c2656b5bee..0000000000 --- a/curricula/cs336_a1/modules/swiglu/bugs/missing_gate_draft_v2.json +++ /dev/null @@ -1,70 +0,0 @@ -{ - "id": "swiglu-missing-gate", - "description": "Missing gate computation in SwiGLU", - "injection_type": "ast", - "engine_version": "2.1", - "target_function": "forward", - "logic": [ - { - "pass": 1, - "type": "find_and_replace", - "description": "Delete gate = self.w3(x)", - "pattern": { - "node_type": "Assign", - "targets": [ - { - "node_type": "Name", - "id": "gate" - } - ], - "value": { - "node_type": "Call" - } - }, - "replacement": { - "type": "delete_statement" - } - }, - { - "pass": 2, - "type": "find_and_replace", - "description": "Delete gated = value * gate", - "pattern": { - "node_type": "Assign", - "targets": [ - { - "node_type": "Name", - "id": "gated" - } - ], - "value": { - "node_type": "BinOp", - "op": "Mult" - } - }, - "replacement": { - "type": "delete_statement" - } - }, - { - "pass": 3, - "type": "find_and_replace", - "description": "Replace return with just value", - "pattern": { - "node_type": "Return", - "value": { - "node_type": "Call" - } - }, - "replacement": { - "type": "replace_value_with", - "source": "value" - } - } - ], - "metadata": { - "created": "2025-11-14", - "version": "2.1", - "author": "auto_fixed" - } -} \ No newline at end of file diff --git a/curricula/cs336_a1/modules/swiglu/bugs/missing_gate_symptom.txt b/curricula/cs336_a1/modules/swiglu/bugs/missing_gate_symptom.txt new file mode 100644 index 0000000000..b0963d6241 --- /dev/null +++ b/curricula/cs336_a1/modules/swiglu/bugs/missing_gate_symptom.txt @@ -0,0 +1,14 @@ +# Bug Symptom: Swiglu — swiglu-missing-gate + +## Observed Behavior +A subtle semantic bug has been injected into your `forward` implementation. Your code +runs without raising a syntax error, but the module's correctness test now fails: +Gate branch reuses w1 instead of w3, so the SwiGLU gating is wrong. + +## Your Challenge +A single line in `cs336_basics/layers.py` no longer matches your correct implementation. Read the +function, form a hypothesis from the symptom above, and locate the divergence. +Restore the correct behavior — do not rewrite the whole function. + +## Verify +Run `mastery submit` to re-run the validator once you believe the bug is fixed. diff --git a/curricula/cs336_a1/modules/text_generation/bugs/temperature_after_softmax.json b/curricula/cs336_a1/modules/text_generation/bugs/temperature_after_softmax.json deleted file mode 100644 index 7562ed59fc..0000000000 --- a/curricula/cs336_a1/modules/text_generation/bugs/temperature_after_softmax.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "id": "text-generation-temperature-after-softmax", - "description": "Temperature applied after softmax", - "injection_type": "ast", - "engine_version": "2.1", - "target_function": "generate", - "logic": [ - { - "pass": 1, - "type": "find_and_replace", - "pattern": { - "node_type": "Assign", - "targets": [ - { - "node_type": "Name", - "id": "probs" - } - ] - }, - "replacement": { - "type": "replace_value_with", - "source": "F.softmax(next_logits / temperature, dim=-1)" - } - } - ] -} \ No newline at end of file diff --git a/curricula/cs336_a1/modules/text_generation/bugs/temperature_after_softmax.patch b/curricula/cs336_a1/modules/text_generation/bugs/temperature_after_softmax.patch deleted file mode 100644 index 69f411826a..0000000000 --- a/curricula/cs336_a1/modules/text_generation/bugs/temperature_after_softmax.patch +++ /dev/null @@ -1,18 +0,0 @@ ---- a/cs336_basics/generation.py -+++ b/cs336_basics/generation.py -@@ -35,11 +35,12 @@ def generate( - # Get logits for next token - logits = model(tokens) # (1, seq_len, vocab_size) - next_logits = logits[0, -1, :] # (vocab_size,) - -- # Apply temperature to logits BEFORE softmax -- next_logits = next_logits / temperature -- probs = F.softmax(next_logits, dim=-1) -+ # BUG: Applying temperature AFTER softmax instead of before! -+ # This doesn't properly scale the distribution - temperature must scale logits. -+ # Should be: next_logits = next_logits / temperature; probs = F.softmax(next_logits) -+ probs = F.softmax(next_logits, dim=-1) -+ probs = probs / temperature # Wrong! Temperature after softmax has no effect! - - # Apply sampling strategy - if top_p is not None: diff --git a/curricula/cs336_a1/modules/text_generation/bugs/temperature_after_softmax_draft.json b/curricula/cs336_a1/modules/text_generation/bugs/temperature_after_softmax_draft.json deleted file mode 100644 index ae9179046b..0000000000 --- a/curricula/cs336_a1/modules/text_generation/bugs/temperature_after_softmax_draft.json +++ /dev/null @@ -1,52 +0,0 @@ -{ - "id": "text_generation-temperature-bug", - "description": "Applies temperature after softmax instead of before, causing incorrect scaling of the distribution.", - "injection_type": "ast", - "engine_version": "2.1", - "target_function": "generate", - "logic": [ - { - "pass_": 1, - "type": "find_and_replace", - "description": "Find softmax application and replace with temperature scaling before softmax.", - "pattern": { - "node_type": "Assign", - "targets": [ - { - "node_type": "Name", - "id": "probs" - } - ], - "value": { - "node_type": "Call", - "op": null, - "attr": null - }, - "attr": null, - "op": null, - "func": { - "node_type": "Attribute", - "op": null, - "attr": "softmax" - }, - "left": null, - "right": null, - "args": null, - "keywords": null - }, - "conditions": null, - "track_as": null, - "replacement": { - "type": "replace_value_with", - "source": "F.softmax(next_logits / temperature, dim=-1)", - "name": null - } - } - ], - "metadata": { - "created": "2025-11-13", - "version": "2.0", - "author": "LLM-Generated", - "tier": "simple" - } -} \ No newline at end of file diff --git a/curricula/cs336_a1/modules/tokenizer_class/bugs/wrong_advance.json b/curricula/cs336_a1/modules/tokenizer_class/bugs/wrong_advance.json new file mode 100644 index 0000000000..28e4070fc0 --- /dev/null +++ b/curricula/cs336_a1/modules/tokenizer_class/bugs/wrong_advance.json @@ -0,0 +1,31 @@ +{ + "id": "tokenizer-class-wrong-advance", + "description": "Special-token cursor advances one char too far (len(tok)+1).", + "injection_type": "ast", + "engine_version": "2.1", + "target_function": "encode", + "logic": [ + { + "pass": 1, + "type": "find_and_replace", + "pattern": { + "node_type": "AugAssign", + "target": { + "node_type": "Name", + "id": "i" + }, + "value": { + "node_type": "Call", + "func": { + "node_type": "Name", + "id": "len" + } + } + }, + "replacement": { + "type": "replace_value_with", + "source": "len(tok) + 1" + } + } + ] +} diff --git a/curricula/cs336_a1/modules/tokenizer_class/bugs/wrong_advance_symptom.txt b/curricula/cs336_a1/modules/tokenizer_class/bugs/wrong_advance_symptom.txt new file mode 100644 index 0000000000..771240020f --- /dev/null +++ b/curricula/cs336_a1/modules/tokenizer_class/bugs/wrong_advance_symptom.txt @@ -0,0 +1,14 @@ +# Bug Symptom: Tokenizer Class — tokenizer-class-wrong-advance + +## Observed Behavior +A subtle semantic bug has been injected into your `encode` implementation. Your code +runs without raising a syntax error, but the module's correctness test now fails: +Special-token cursor advances one char too far (len(tok)+1). + +## Your Challenge +A single line in `cs336_basics/tokenizer.py` no longer matches your correct implementation. Read the +function, form a hypothesis from the symptom above, and locate the divergence. +Restore the correct behavior — do not rewrite the whole function. + +## Verify +Run `mastery submit` to re-run the validator once you believe the bug is fixed. diff --git a/curricula/cs336_a1/modules/tokenizer_class/bugs/wrong_merge_order.json b/curricula/cs336_a1/modules/tokenizer_class/bugs/wrong_merge_order.json deleted file mode 100644 index 9496b2b0b2..0000000000 --- a/curricula/cs336_a1/modules/tokenizer_class/bugs/wrong_merge_order.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "id": "tokenizer-class-wrong-merge-order", - "description": "Merges applied in reversed order", - "injection_type": "ast", - "engine_version": "2.1", - "target_function": "encode", - "logic": [{ - "pass": 1, - "type": "find_and_replace", - "pattern": { - "node_type": "For", - "target": {"node_type": "Tuple"}, - "iter": {"node_type": "Attribute", "attr": "merges"} - }, - "replacement": { - "type": "replace_with", - "source": "for pair_a, pair_b in reversed(self.merges):\n tokens = self._apply_merge(tokens, pair_a, pair_b)" - } - }] -} \ No newline at end of file diff --git a/curricula/cs336_a1/modules/tokenizer_class/bugs/wrong_merge_order.patch b/curricula/cs336_a1/modules/tokenizer_class/bugs/wrong_merge_order.patch deleted file mode 100644 index 937b3030c9..0000000000 --- a/curricula/cs336_a1/modules/tokenizer_class/bugs/wrong_merge_order.patch +++ /dev/null @@ -1,15 +0,0 @@ ---- a/cs336_basics/tokenizer.py -+++ b/cs336_basics/tokenizer.py -@@ -15,8 +15,10 @@ class Tokenizer: - # Start with byte-level tokens - tokens = list(text_bytes) - -- # Apply each merge in order -- for pair_a, pair_b in self.merges: -+ # BUG: Applying merges in random/wrong order! -+ # Merges must be applied in the EXACT order they were learned -+ # Wrong order produces different encodings -+ for pair_a, pair_b in reversed(self.merges): # REVERSED - WRONG! - tokens = self._apply_merge(tokens, pair_a, pair_b) - - return [self.vocab_r[t] for t in tokens] diff --git a/curricula/cs336_a1/modules/tokenizer_class/bugs/wrong_merge_order_draft.json b/curricula/cs336_a1/modules/tokenizer_class/bugs/wrong_merge_order_draft.json deleted file mode 100644 index 3cdd388591..0000000000 --- a/curricula/cs336_a1/modules/tokenizer_class/bugs/wrong_merge_order_draft.json +++ /dev/null @@ -1,65 +0,0 @@ -{ - "id": "tokenizer-class-bug", - "description": "Bug in tokenizer_class where merges are applied in reverse order, causing incorrect tokenization.", - "injection_type": "ast", - "engine_version": "2.1", - "target_function": "tokenizer_class", - "logic": [ - { - "pass_": 1, - "type": "find_and_replace", - "description": "Find the for loop applying merges and reverse the order of merges.", - "pattern": { - "node_type": "For", - "targets": [ - { - "node_type": "Name", - "id": "pair_a" - }, - { - "node_type": "Name", - "id": "pair_b" - } - ], - "value": { - "node_type": "Attribute", - "op": null, - "attr": "merges" - }, - "attr": null, - "op": null, - "func": null, - "left": null, - "right": null, - "args": null, - "keywords": null - }, - "conditions": [ - { - "check": "target_is_name", - "value": null, - "index": 0, - "name": "pair_a" - }, - { - "check": "target_is_name", - "value": null, - "index": 1, - "name": "pair_b" - } - ], - "track_as": null, - "replacement": { - "type": "replace_value_with", - "source": "reversed(self.merges)", - "name": null - } - } - ], - "metadata": { - "created": "2025-11-13", - "version": "2.0", - "author": "LLM-Generated", - "tier": "simple" - } -} \ No newline at end of file diff --git a/curricula/cs336_a1/modules/tokenizer_class/validator.sh b/curricula/cs336_a1/modules/tokenizer_class/validator.sh index 2809affd5f..ab91b1a21a 100755 --- a/curricula/cs336_a1/modules/tokenizer_class/validator.sh +++ b/curricula/cs336_a1/modules/tokenizer_class/validator.sh @@ -20,12 +20,12 @@ start_time=$(python3 -c 'import time; print(time.time())') if [ -n "$MASTERY_PYTHON" ]; then export PYTHONPATH="$(pwd):$PYTHONPATH" - "$MASTERY_PYTHON" -m pytest tests/test_tokenizer.py::test_tokenizer_class -v --tb=short --import-mode=importlib + "$MASTERY_PYTHON" -m pytest tests/test_tokenizer.py::test_overlapping_special_tokens -v --tb=short --import-mode=importlib elif [ -n "$VIRTUAL_ENV" ]; then export PYTHONPATH="$(pwd):$PYTHONPATH" - "$VIRTUAL_ENV/bin/python" -m pytest tests/test_tokenizer.py::test_tokenizer_class -v --tb=short --import-mode=importlib + "$VIRTUAL_ENV/bin/python" -m pytest tests/test_tokenizer.py::test_overlapping_special_tokens -v --tb=short --import-mode=importlib else - uv run pytest tests/test_tokenizer.py::test_tokenizer_class -v --tb=short --import-mode=importlib + uv run pytest tests/test_tokenizer.py::test_overlapping_special_tokens -v --tb=short --import-mode=importlib fi end_time=$(python3 -c 'import time; print(time.time())') diff --git a/curricula/cs336_a1/modules/training_loop/bugs/missing_zero_grad.json b/curricula/cs336_a1/modules/training_loop/bugs/missing_zero_grad.json deleted file mode 100644 index e7f05d8435..0000000000 --- a/curricula/cs336_a1/modules/training_loop/bugs/missing_zero_grad.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "id": "training-loop-missing-zero-grad", - "description": "Missing optimizer.zero_grad()", - "injection_type": "ast", - "engine_version": "2.1", - "target_function": "train_step", - "logic": [ - { - "pass": 1, - "type": "find_and_replace", - "pattern": { - "node_type": "Expr", - "value": { - "node_type": "Call", - "func": { - "node_type": "Attribute", - "attr": "zero_grad" - } - } - }, - "replacement": { - "type": "delete_statement" - } - } - ] -} \ No newline at end of file diff --git a/curricula/cs336_a1/modules/training_loop/bugs/missing_zero_grad.patch b/curricula/cs336_a1/modules/training_loop/bugs/missing_zero_grad.patch deleted file mode 100644 index bd8f4038fe..0000000000 --- a/curricula/cs336_a1/modules/training_loop/bugs/missing_zero_grad.patch +++ /dev/null @@ -1,14 +0,0 @@ ---- a/cs336_basics/training.py -+++ b/cs336_basics/training.py -@@ -45,8 +45,9 @@ def train_loop( - loss = F.cross_entropy(logits.view(-1, vocab_size), targets.view(-1)) - - # Backward pass -- optimizer.zero_grad() # Clear previous gradients -- loss.backward() # Compute new gradients -+ # BUG: Missing optimizer.zero_grad()! Gradients accumulate across steps, -+ # growing unbounded and causing parameter explosion within a few iterations. -+ loss.backward() # Should call optimizer.zero_grad() BEFORE this! - - # Gradient clipping - torch.nn.utils.clip_grad_norm_(model.parameters(), max_grad_norm) diff --git a/curricula/cs336_a1/modules/training_loop/bugs/missing_zero_grad_draft.json b/curricula/cs336_a1/modules/training_loop/bugs/missing_zero_grad_draft.json deleted file mode 100644 index 96fe753ece..0000000000 --- a/curricula/cs336_a1/modules/training_loop/bugs/missing_zero_grad_draft.json +++ /dev/null @@ -1,47 +0,0 @@ -{ - "id": "training_loop-missing-zero-grad", - "description": "Removes the optimizer.zero_grad() call, causing gradients to accumulate and grow unbounded.", - "injection_type": "ast", - "engine_version": "2.1", - "target_function": "training_loop", - "logic": [ - { - "pass_": 1, - "type": "find_and_replace", - "description": "Delete the optimizer.zero_grad() call before loss.backward()", - "pattern": { - "node_type": "Expr", - "targets": null, - "value": { - "node_type": "Call", - "op": null, - "attr": null - }, - "attr": null, - "op": null, - "func": { - "node_type": "Attribute", - "op": null, - "attr": "zero_grad" - }, - "left": null, - "right": null, - "args": null, - "keywords": null - }, - "conditions": null, - "track_as": null, - "replacement": { - "type": "delete_statement", - "source": null, - "name": null - } - } - ], - "metadata": { - "created": "2025-11-13", - "version": "2.0", - "author": "LLM-Generated", - "tier": "simple" - } -} \ No newline at end of file diff --git a/curricula/cs336_a1/modules/training_loop/bugs/missing_zero_grad_draft_v2.json b/curricula/cs336_a1/modules/training_loop/bugs/missing_zero_grad_draft_v2.json deleted file mode 100644 index 555ebeffd0..0000000000 --- a/curricula/cs336_a1/modules/training_loop/bugs/missing_zero_grad_draft_v2.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "id": "training-loop-missing-zero-grad", - "description": "Missing optimizer.zero_grad() in training loop", - "injection_type": "ast", - "engine_version": "2.1", - "target_function": "train_step", - "logic": [ - { - "pass": 1, - "type": "find_and_replace", - "description": "Delete optimizer.zero_grad() call", - "pattern": { - "node_type": "Expr", - "value": { - "node_type": "Call", - "func": { - "node_type": "Attribute", - "attr": "zero_grad" - } - } - }, - "replacement": { - "type": "delete_statement" - } - } - ], - "metadata": { - "created": "2025-11-14", - "version": "2.1", - "author": "auto_fixed" - } -} \ No newline at end of file diff --git a/curricula/cs336_a1/modules/transformer_block/bugs/missing_residual.json b/curricula/cs336_a1/modules/transformer_block/bugs/missing_residual.json index e42f4eb197..1e7af85bf2 100644 --- a/curricula/cs336_a1/modules/transformer_block/bugs/missing_residual.json +++ b/curricula/cs336_a1/modules/transformer_block/bugs/missing_residual.json @@ -1,9 +1,9 @@ { "id": "transformer-block-missing-residual", - "description": "Missing residual connection", + "description": "Drop the attention residual add (x = attn_out instead of x + attn_out).", "injection_type": "ast", "engine_version": "2.1", - "target_function": "forward", + "target_function": "transformer_block", "logic": [ { "pass": 1, @@ -18,7 +18,10 @@ ], "value": { "node_type": "BinOp", - "op": "Add" + "right": { + "node_type": "Name", + "id": "attn_out" + } } }, "replacement": { @@ -27,4 +30,4 @@ } } ] -} \ No newline at end of file +} diff --git a/curricula/cs336_a1/modules/transformer_block/bugs/missing_residual.patch b/curricula/cs336_a1/modules/transformer_block/bugs/missing_residual.patch deleted file mode 100644 index b281aed7e8..0000000000 --- a/curricula/cs336_a1/modules/transformer_block/bugs/missing_residual.patch +++ /dev/null @@ -1,12 +0,0 @@ ---- a/cs336_basics/layers.py -+++ b/cs336_basics/layers.py -@@ -450,7 +450,8 @@ class TransformerBlock(nn.Module): - ) - - # Residual connection -- x = x + attn_out -+ # BUG: Missing residual connection! Without this, gradient highway is broken. -+ x = attn_out # Should be: x = x + attn_out - - # Feed-forward sublayer - normed = self.norm2(x) diff --git a/curricula/cs336_a1/modules/transformer_block/bugs/missing_residual_draft.json b/curricula/cs336_a1/modules/transformer_block/bugs/missing_residual_draft.json deleted file mode 100644 index 1d3730c6cf..0000000000 --- a/curricula/cs336_a1/modules/transformer_block/bugs/missing_residual_draft.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "id": "transformer_block-bug", - "description": "Bug in transformer_block where x is incorrectly assigned to attn_out instead of x + attn_out.", - "injection_type": "ast", - "engine_version": "2.1", - "target_function": "transformer_block", - "logic": [ - { - "pass_": 1, - "type": "find_and_replace", - "description": "Find x = x + attn_out pattern and replace with x = attn_out", - "pattern": { - "node_type": "Assign", - "targets": [ - { - "node_type": "Name", - "id": "x" - } - ], - "value": { - "node_type": "BinOp", - "op": "Add", - "attr": null - }, - "attr": null, - "op": null, - "func": null, - "left": null, - "right": null, - "args": null, - "keywords": null - }, - "conditions": null, - "track_as": null, - "replacement": { - "type": "replace_value_with", - "source": "attn_out", - "name": null - } - } - ], - "metadata": { - "created": "2023-11-24", - "version": "2.0", - "author": "LLM-Generated", - "tier": "simple" - } -} \ No newline at end of file diff --git a/curricula/cs336_a1/modules/transformer_block/bugs/missing_residual_draft_v2.json b/curricula/cs336_a1/modules/transformer_block/bugs/missing_residual_draft_v2.json deleted file mode 100644 index 040c07b1dc..0000000000 --- a/curricula/cs336_a1/modules/transformer_block/bugs/missing_residual_draft_v2.json +++ /dev/null @@ -1,36 +0,0 @@ -{ - "id": "transformer-block-missing-residual", - "description": "Missing residual connection in transformer block", - "injection_type": "ast", - "engine_version": "2.1", - "target_function": "forward", - "logic": [ - { - "pass": 1, - "type": "find_and_replace", - "description": "Replace x = x + attn_out with x = attn_out", - "pattern": { - "node_type": "Assign", - "targets": [ - { - "node_type": "Name", - "id": "x" - } - ], - "value": { - "node_type": "BinOp", - "op": "Add" - } - }, - "replacement": { - "type": "replace_value_with", - "source": "attn_out" - } - } - ], - "metadata": { - "created": "2025-11-14", - "version": "2.1", - "author": "auto_fixed" - } -} \ No newline at end of file diff --git a/curricula/cs336_a1/modules/transformer_block/bugs/missing_residual_symptom.txt b/curricula/cs336_a1/modules/transformer_block/bugs/missing_residual_symptom.txt new file mode 100644 index 0000000000..3ca5d65af9 --- /dev/null +++ b/curricula/cs336_a1/modules/transformer_block/bugs/missing_residual_symptom.txt @@ -0,0 +1,14 @@ +# Bug Symptom: Transformer Block — transformer-block-missing-residual + +## Observed Behavior +A subtle semantic bug has been injected into your `transformer_block` implementation. Your code +runs without raising a syntax error, but the module's correctness test now fails: +Drop the attention residual add (x = attn_out instead of x + attn_out). + +## Your Challenge +A single line in `cs336_basics/layers.py` no longer matches your correct implementation. Read the +function, form a hypothesis from the symptom above, and locate the divergence. +Restore the correct behavior — do not rewrite the whole function. + +## Verify +Run `mastery submit` to re-run the validator once you believe the bug is fixed. diff --git a/curricula/cs336_a1/modules/transformer_lm/bugs/missing_final_norm.json b/curricula/cs336_a1/modules/transformer_lm/bugs/missing_final_norm.json index 9024dfad8a..810505441c 100644 --- a/curricula/cs336_a1/modules/transformer_lm/bugs/missing_final_norm.json +++ b/curricula/cs336_a1/modules/transformer_lm/bugs/missing_final_norm.json @@ -1,20 +1,33 @@ { "id": "transformer-lm-missing-final-norm", - "description": "Missing final layer normalization before LM head", + "description": "Skip the final RMSNorm before the LM head (x = x instead of ln_final(x)).", "injection_type": "ast", "engine_version": "2.1", - "target_function": "forward", - "logic": [{ - "pass": 1, - "type": "find_and_replace", - "pattern": { - "node_type": "Assign", - "targets": [{"node_type": "Name", "id": "x"}], - "value": { - "node_type": "Call", - "func": {"node_type": "Attribute", "attr": "ln_final"} + "target_function": "transformer_lm", + "logic": [ + { + "pass": 1, + "type": "find_and_replace", + "pattern": { + "node_type": "Assign", + "targets": [ + { + "node_type": "Name", + "id": "x" + } + ], + "value": { + "node_type": "Call", + "func": { + "node_type": "Name", + "id": "ln_final" + } + } + }, + "replacement": { + "type": "replace_value_with", + "source": "x" } - }, - "replacement": {"type": "delete_statement"} - }] -} \ No newline at end of file + } + ] +} diff --git a/curricula/cs336_a1/modules/transformer_lm/bugs/missing_final_norm.patch b/curricula/cs336_a1/modules/transformer_lm/bugs/missing_final_norm.patch deleted file mode 100644 index 4b3afb7058..0000000000 --- a/curricula/cs336_a1/modules/transformer_lm/bugs/missing_final_norm.patch +++ /dev/null @@ -1,17 +0,0 @@ ---- a/cs336_basics/layers.py -+++ b/cs336_basics/layers.py -@@ -40,10 +40,8 @@ def transformer_lm(...): - in_features=x, - ) - -- # Final RMSNorm -- ln_final = RMSNorm(d_model=d_model, eps=1e-5) -- with torch.no_grad(): -- ln_final.weight.copy_(weights["ln_final.weight"]) -- x = ln_final(x) -+ # BUG: Missing final normalization! -+ # Without this, hidden states have large variance before LM head -+ # This causes unstable softmax and poor gradient flow -+ # Should apply: x = RMSNorm(x) using weights["ln_final.weight"] - - # LM head projection diff --git a/curricula/cs336_a1/modules/transformer_lm/bugs/missing_final_norm_draft.json b/curricula/cs336_a1/modules/transformer_lm/bugs/missing_final_norm_draft.json deleted file mode 100644 index 8a206ee2a6..0000000000 --- a/curricula/cs336_a1/modules/transformer_lm/bugs/missing_final_norm_draft.json +++ /dev/null @@ -1,47 +0,0 @@ -{ - "id": "silu-missing-multiply", - "description": "Removes the multiplication by input, returning only sigmoid(x) instead of x * sigmoid(x).", - "injection_type": "ast", - "engine_version": "2.1", - "target_function": "silu", - "logic": [ - { - "pass_": 1, - "type": "find_and_replace", - "description": "Find x * sigmoid(x) pattern and replace with just sigmoid(x)", - "pattern": { - "node_type": "BinOp", - "targets": null, - "value": null, - "attr": null, - "op": "Mult", - "func": null, - "left": { - "node_type": "Name", - "op": null, - "attr": null - }, - "right": { - "node_type": "Call", - "op": null, - "attr": null - }, - "args": null, - "keywords": null - }, - "conditions": null, - "track_as": null, - "replacement": { - "type": "replace_with", - "source": "node.right", - "name": null - } - } - ], - "metadata": { - "created": "2025-11-13", - "version": "2.0", - "author": "LLM-Generated", - "tier": "simple" - } -} \ No newline at end of file diff --git a/curricula/cs336_a1/modules/transformer_lm/bugs/missing_final_norm_symptom.txt b/curricula/cs336_a1/modules/transformer_lm/bugs/missing_final_norm_symptom.txt new file mode 100644 index 0000000000..e9e97a36d7 --- /dev/null +++ b/curricula/cs336_a1/modules/transformer_lm/bugs/missing_final_norm_symptom.txt @@ -0,0 +1,14 @@ +# Bug Symptom: Transformer Lm — transformer-lm-missing-final-norm + +## Observed Behavior +A subtle semantic bug has been injected into your `transformer_lm` implementation. Your code +runs without raising a syntax error, but the module's correctness test now fails: +Skip the final RMSNorm before the LM head (x = x instead of ln_final(x)). + +## Your Challenge +A single line in `cs336_basics/layers.py` no longer matches your correct implementation. Read the +function, form a hypothesis from the symptom above, and locate the divergence. +Restore the correct behavior — do not rewrite the whole function. + +## Verify +Run `mastery submit` to re-run the validator once you believe the bug is fixed. diff --git a/engine/ast_harden/pattern_matcher.py b/engine/ast_harden/pattern_matcher.py index 5d6cdaf743..6869aaa0f1 100644 --- a/engine/ast_harden/pattern_matcher.py +++ b/engine/ast_harden/pattern_matcher.py @@ -389,13 +389,13 @@ def _create_replacement(self, node: ast.AST) -> Optional[ast.AST]: node ) else: - # Generic fallback: try to copy all attributes - new_node = node.__class__() - for attr in node._fields: - if attr == 'value': - setattr(new_node, attr, new_value) - elif hasattr(node, attr): - setattr(new_node, attr, getattr(node, attr)) + # Generic fallback: shallow-copy the node (preserves required + # fields like AugAssign.op/target) then swap in the new value. + # Avoids the empty `node.__class__()` constructor, which emits a + # DeprecationWarning (and is a hard error from Python 3.15). + import copy as _copy + new_node = _copy.copy(node) + new_node.value = new_value new_node = ast.copy_location(new_node, node) return new_node diff --git a/engine/schemas.py b/engine/schemas.py index 5b1587e1a9..880bdf9634 100644 --- a/engine/schemas.py +++ b/engine/schemas.py @@ -11,7 +11,7 @@ from enum import Enum from typing import Optional, List -from pydantic import BaseModel, Field +from pydantic import BaseModel, ConfigDict, Field class CurriculumType(str, Enum): @@ -243,8 +243,7 @@ class ContextReference(BaseModel): """Reference to a tracked context variable""" from_context: str - class Config: - extra = 'forbid' + model_config = ConfigDict(extra='forbid') class NameNode(BaseModel): @@ -252,8 +251,7 @@ class NameNode(BaseModel): node_type: Literal["Name"] id: Optional[Union[str, ContextReference]] = None - class Config: - extra = 'forbid' + model_config = ConfigDict(extra='forbid') class NestedPattern(BaseModel): @@ -262,8 +260,7 @@ class NestedPattern(BaseModel): op: Optional[str] = None attr: Optional[str] = None - class Config: - extra = 'forbid' + model_config = ConfigDict(extra='forbid') class KeywordArg(BaseModel): @@ -271,14 +268,14 @@ class KeywordArg(BaseModel): arg: Optional[str] = None value: Optional[NestedPattern] = None - class Config: - extra = 'forbid' + model_config = ConfigDict(extra='forbid') class Pattern(BaseModel): """AST pattern for matching nodes""" node_type: str targets: Optional[List[NameNode]] = None + target: Optional[NameNode] = None # singular target, e.g. for AugAssign nodes value: Optional[NestedPattern] = None attr: Optional[str] = None op: Optional[str] = None @@ -288,8 +285,7 @@ class Pattern(BaseModel): args: Optional[List[NestedPattern]] = None keywords: Optional[List[KeywordArg]] = None - class Config: - extra = 'forbid' + model_config = ConfigDict(extra='forbid') class Condition(BaseModel): @@ -299,8 +295,7 @@ class Condition(BaseModel): index: Optional[int] = None name: Optional[str] = None - class Config: - extra = 'forbid' + model_config = ConfigDict(extra='forbid') class Replacement(BaseModel): @@ -309,23 +304,20 @@ class Replacement(BaseModel): source: Optional[Union[str, ContextReference]] = None name: Optional[str] = None - class Config: - extra = 'forbid' + model_config = ConfigDict(extra='forbid') class PassDefinition(BaseModel): """Single pass in bug injection logic""" pass_: int = Field(..., alias="pass") type: str - description: str + description: Optional[str] = None # shipped defs omit per-pass descriptions pattern: Optional[Pattern] = None conditions: Optional[List[Condition]] = None track_as: Optional[Dict[str, str]] = None replacement: Optional[Replacement] = None - class Config: - populate_by_name = True - extra = 'forbid' + model_config = ConfigDict(populate_by_name=True, extra='forbid') class BugMetadata(BaseModel): @@ -335,8 +327,7 @@ class BugMetadata(BaseModel): author: str tier: str - class Config: - extra = 'forbid' + model_config = ConfigDict(extra='forbid') class BugDefinition(BaseModel): @@ -347,7 +338,9 @@ class BugDefinition(BaseModel): engine_version: str target_function: str logic: List[PassDefinition] - metadata: BugMetadata + # Optional: hand-authored/shipped bug defs omit this; only the LLM authoring + # tool (bug_author) populates it. Keeping it required would reject every + # shipped definition. + metadata: Optional[BugMetadata] = None - class Config: - extra = 'forbid' + model_config = ConfigDict(extra='forbid') diff --git a/pyproject.toml b/pyproject.toml index 09ecd8f4d2..6f454b0869 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -50,7 +50,8 @@ log_cli = true log_cli_level = "WARNING" addopts = "-s" markers = [ - "integration: marks tests as integration tests that use real API calls (deselect with '-m \"not integration\"')" + "integration: marks tests as integration tests that use real API calls (deselect with '-m \"not integration\"')", + "slow: marks tests as slow-running (deselect with '-m \"not slow\"')" ] [tool.ruff] diff --git a/scripts/check_bug_defs.py b/scripts/check_bug_defs.py new file mode 100644 index 0000000000..d8d24db6c6 --- /dev/null +++ b/scripts/check_bug_defs.py @@ -0,0 +1,125 @@ +#!/usr/bin/env python3 +"""Meta-test harness: verify every active cs336 bug definition injects against the reference. + +For each bug def under curricula/cs336_a1/modules/*/bugs/*.json (optionally excluding +*_draft / *_v2), resolve the reference source file that defines the def's +`target_function` (preferring developer reference, falling back to the student build +target), run GenericBugInjector, and report whether injection succeeds. + +A bug def is HEALTHY iff the injector returns success (target found AND a pattern matched). +This is the Phase-0 "silent bug" guard for the Harden stage. + +Usage: + python scripts/check_bug_defs.py # active defs only (skip _draft/_v2) + python scripts/check_bug_defs.py --all # include drafts/v2 +""" +from __future__ import annotations + +import ast +import json +import sys +from pathlib import Path + +# Make `engine` importable when run directly. +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from engine.ast_harden.generic_injector import GenericBugInjector # noqa: E402 + +REPO = Path(__file__).resolve().parents[1] +DEV = REPO / "modes" / "developer" / "cs336_basics" +STU = REPO / "modes" / "student" / "cs336_basics" +MODULES = REPO / "curricula" / "cs336_a1" / "modules" + + +def reference_files() -> list[Path]: + """Developer reference files first, then student-only files (e.g. generation.py).""" + ordered = sorted(DEV.glob("*.py")) + for p in sorted(STU.glob("*.py")): + if not (DEV / p.name).exists(): + ordered.append(p) + return ordered + + +def _defines(src: str, name: str) -> bool: + """True if src defines a function/method named `name` (methods count via ast.walk).""" + try: + tree = ast.parse(src) + except SyntaxError: + return False + return any( + isinstance(n, (ast.FunctionDef, ast.AsyncFunctionDef)) and n.name == name + for n in ast.walk(tree) + ) + + +def resolve_reference(target: str) -> tuple[Path | None, str | None]: + for p in reference_files(): + src = p.read_text(encoding="utf-8") + if _defines(src, target): + return p, src + return None, None + + +def is_active(path: Path) -> bool: + stem = path.stem + return not (stem.endswith("_draft") or stem.endswith("_v2") or "_draft" in stem) + + +def check(path: Path) -> tuple[str, str]: + """Return (status, detail). status in INJECTED_OK / PATTERN_MISS / NO_TARGET_FN / DEF_ERROR.""" + try: + bug = json.loads(path.read_text(encoding="utf-8")) + except Exception as e: # noqa: BLE001 + return "DEF_ERROR", f"json: {e}" + target = bug.get("target_function") + if not target: + return "DEF_ERROR", "no target_function" + ref, src = resolve_reference(target) + if ref is None: + return "NO_TARGET_FN", f"'{target}' not defined in any reference file" + try: + buggy_src, ok = GenericBugInjector(bug).inject(src) + except Exception as e: # noqa: BLE001 + return "DEF_ERROR", f"{type(e).__name__}: {e}" + if not ok: + return "PATTERN_MISS", f"target '{target}' in {ref.name}, but no node matched" + # inject() can return ok=True yet leave the source unchanged; a no-op "match" is + # not a real bug, so hold it to the same bar as test_bug_defs_match.py. + if ast.dump(ast.parse(buggy_src)) == ast.dump(ast.parse(src)): + return "PATTERN_MISS", f"target '{target}' in {ref.name}, but injection was a no-op" + return "INJECTED_OK", f"{target} in {ref.name}" + + +def collect(include_all: bool) -> list[tuple[str, Path]]: + out = [] + for jp in sorted(MODULES.glob("*/bugs/*.json")): + if include_all or is_active(jp): + out.append((jp.parent.parent.name, jp)) + return out + + +def main(argv: list[str]) -> int: + include_all = "--all" in argv + rows = [] + for module, jp in collect(include_all): + status, detail = check(jp) + rows.append((module, jp.name, status, detail)) + + width = max((len(m) for m, *_ in rows), default=6) + print(f"{'MODULE':<{width}} {'BUG FILE':<34} {'STATUS':<13} DETAIL") + print("-" * (width + 90)) + for module, name, status, detail in rows: + print(f"{module:<{width}} {name:<34} {status:<13} {detail}") + print("-" * (width + 90)) + + from collections import Counter + + tally = Counter(r[2] for r in rows) + ok = tally.get("INJECTED_OK", 0) + print(f"SUMMARY: {dict(tally)} | healthy={ok}/{len(rows)}") + # Exit non-zero if any active def is unhealthy. + return 0 if ok == len(rows) else 1 + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv[1:])) diff --git a/tests/e2e/conftest.py b/tests/e2e/conftest.py new file mode 100644 index 0000000000..e32e1223ee --- /dev/null +++ b/tests/e2e/conftest.py @@ -0,0 +1,33 @@ +"""Shared fixtures for the end-to-end suite. + +These E2E tests drive the real engine (in-process via CliRunner and out-of-process +via subprocess). Two cross-cutting concerns are handled here for every E2E test: + +1. HOME isolation — the engine persists progress to `~/.mastery_progress.json` and + the cognitive-evidence ledger to `~/.mastery_evidence.jsonl` (both via Path.home()). + Without isolation, running the suite would read/overwrite the developer's REAL + progress file. We redirect HOME (honored by Path.home() on POSIX) to a per-test + temp dir so the suite is non-destructive by construction. + +2. Wide console — the engine renders Rich panels whose width follows the COLUMNS env + var when stdout is captured (not a TTY). At the default 80 cols, long temp paths + wrap across panel borders and break substring assertions on stdout. We force a wide + console so paths/messages stay on one line. +""" +import pytest + +from engine.state import StateManager + + +@pytest.fixture(autouse=True) +def isolated_home_and_wide_console(tmp_path, monkeypatch): + home = tmp_path / "home" + home.mkdir() + monkeypatch.setenv("HOME", str(home)) + monkeypatch.setenv("COLUMNS", "200") + # StateManager.STATE_FILE is computed from Path.home() at IMPORT time, so setting + # $HOME alone does not redirect it for in-process (CliRunner) tests that already + # imported engine.state. Patch it explicitly so EVERY e2e test — in-process or + # subprocess — uses the isolated progress file, never the developer's real one. + monkeypatch.setattr(StateManager, "STATE_FILE", home / ".mastery_progress.json") + yield diff --git a/tests/e2e/test_adversarial_stress.py b/tests/e2e/test_adversarial_stress.py index 95f941a7f2..3c8635dba5 100644 --- a/tests/e2e/test_adversarial_stress.py +++ b/tests/e2e/test_adversarial_stress.py @@ -165,17 +165,19 @@ def test_corrupted_patch_file(self, isolated_repo: Path): with open(state_file, 'w') as f: json.dump(state, f, indent=2) - # Corrupt the patch file + # Corrupt the active bug definition. The engine now injects bugs via AST + # `.json` definitions (legacy `.patch` files were removed), so the modern + # "corrupted bug asset" scenario is an unparseable bug-definition JSON. bugs_dir = isolated_repo / "curricula/cs336_a1/modules/softmax/bugs" - patch_file = list(bugs_dir.glob("*.patch"))[0] - patch_backup = patch_file.with_suffix('.patch.bak') - patch_file.rename(patch_backup) - - corrupted_patch = '''This is not a valid patch file! -Just random text that will cause the patch command to fail. -No proper patch headers or hunks. + bug_file = list(bugs_dir.glob("*.json"))[0] + bug_backup = bug_file.with_suffix('.json.bak') + bug_file.rename(bug_backup) + + corrupted_bug = '''This is not a valid bug definition! +Just random text that will cause JSON parsing to fail. +No proper injection schema. ''' - patch_file.write_text(corrupted_patch) + bug_file.write_text(corrupted_bug) try: # Try to start harden challenge @@ -186,9 +188,9 @@ def test_corrupted_patch_file(self, isolated_repo: Path): # Should have clear error message assert "error" in result.stdout.lower() or "failed" in result.stdout.lower() finally: - # Restore original patch - patch_file.unlink() - patch_backup.rename(patch_file) + # Restore original bug definition + bug_file.unlink() + bug_backup.rename(bug_file) def test_filesystem_permissions_error(self, isolated_repo: Path, tmp_path: Path): """ diff --git a/tests/e2e/test_complete_bjh_loop.py b/tests/e2e/test_complete_bjh_loop.py index 6cc9da4832..c99a18f722 100644 --- a/tests/e2e/test_complete_bjh_loop.py +++ b/tests/e2e/test_complete_bjh_loop.py @@ -132,6 +132,12 @@ def isolated_repo(tmp_path: Path) -> Generator[Path, None, None]: # Create initial commit (required for git worktree to work) subprocess.run(["git", "add", "-A"], cwd=test_repo, check=True, capture_output=True) + # The copied .gitignore ignores the unanchored name `cs336_basics`, which also + # matches modes/{student,developer}/cs336_basics. In a fresh repo those dirs are + # untracked-and-ignored, so `git add -A` skips them — leaving the shadow + # worktree's cs336_basics symlink dangling. Force-add modes so the reference and + # student implementations are committed and present in the worktree. + subprocess.run(["git", "add", "-f", "modes"], cwd=test_repo, check=True, capture_output=True) subprocess.run( ["git", "commit", "-m", "Initial commit"], cwd=test_repo, check=True, capture_output=True @@ -167,21 +173,40 @@ def isolated_repo(tmp_path: Path) -> Generator[Path, None, None]: # Alternative: Add test_repo to PYTHONPATH for the subprocess # This works even if pip install fails import os + saved_env = {k: os.environ.get(k) for k in ("PYTHONPATH", "HOME", "COLUMNS")} os.environ['PYTHONPATH'] = f"{test_repo}:{os.environ.get('PYTHONPATH', '')}" - - yield test_repo - - # Cleanup: Remove any shadow worktrees before cleaning up directory - shadow_worktree = test_repo / ".mastery_engine_worktree" - if shadow_worktree.exists(): - try: - subprocess.run( - ["git", "worktree", "remove", str(shadow_worktree), "--force"], - cwd=test_repo, - capture_output=True - ) - except subprocess.CalledProcessError: - pass + + # ISOLATION: redirect HOME into the temp repo so the engine's state file + # (~/.mastery_progress.json) and evidence ledger never touch the real home. + # Path.home() honors $HOME on POSIX, so both the subprocess AND this test's + # in-process get_state() read the isolated copy. + fake_home = test_repo / ".home" + fake_home.mkdir(exist_ok=True) + os.environ['HOME'] = str(fake_home) + # Force a wide console so Rich panels don't wrap (paths split across lines + # would otherwise break substring assertions on stdout). + os.environ['COLUMNS'] = "200" + + try: + yield test_repo + finally: + # Cleanup: Remove any shadow worktrees before cleaning up directory + shadow_worktree = test_repo / ".mastery_engine_worktree" + if shadow_worktree.exists(): + try: + subprocess.run( + ["git", "worktree", "remove", str(shadow_worktree), "--force"], + cwd=test_repo, + capture_output=True + ) + except subprocess.CalledProcessError: + pass + # Restore environment we mutated. + for k, v in saved_env.items(): + if v is None: + os.environ.pop(k, None) + else: + os.environ[k] = v def run_engine_command(repo_path: Path, *args: str) -> subprocess.CompletedProcess: diff --git a/tests/e2e/test_full_softmax_loop.py b/tests/e2e/test_full_softmax_loop.py index 7e0231f31f..40a02529b7 100644 --- a/tests/e2e/test_full_softmax_loop.py +++ b/tests/e2e/test_full_softmax_loop.py @@ -40,9 +40,17 @@ def isolated_workspace(tmp_path, monkeypatch): # Set up temporary state file location state_file = tmp_path / ".mastery_progress.json" - # Patch the workspace and state file paths - monkeypatch.setattr("engine.workspace.WorkspaceManager.WORKSPACE_DIR", workspace_dir) + # Patch the state file path. (WorkspaceManager no longer exposes a class-level + # WORKSPACE_DIR — it resolves workspace_root per-instance — so there is nothing + # to patch there; HOME isolation in conftest covers the rest.) monkeypatch.setattr("engine.state.StateManager.STATE_FILE", state_file) + + # The engine's commands call require_shadow_worktree(), which only checks that + # SHADOW_WORKTREE_DIR exists. Point it at an isolated temp dir so in-process + # (CliRunner) tests don't depend on a real worktree in the developer's repo. + worktree_dir = tmp_path / "worktree" + worktree_dir.mkdir() + monkeypatch.setattr("engine.main.SHADOW_WORKTREE_DIR", worktree_dir) return { "workspace": workspace_dir, @@ -84,10 +92,17 @@ def softmax(in_features: Float[torch.Tensor, " ..."], dim: int) -> Float[torch.T return out.to(orig_dtype) """ + @pytest.mark.skip( + reason="Superseded by tests/e2e/test_complete_bjh_loop.py (fully isolated, " + "subprocess-driven, passing). This in-process variant asserts the obsolete " + "patch-based Harden internals (WorkspaceManager.create_harden_workspace / " + "apply_patch) that AST injection replaced, so it can no longer reflect the " + "real loop. Kept (skipped) as a record; the loop is covered by the replacement." + ) def test_complete_softmax_bjh_loop(self, isolated_workspace, mocker): """ Test the complete softmax Build-Justify-Harden loop. - + This is the fortress test that validates all components working together. """ workspace = isolated_workspace["workspace"] @@ -152,6 +167,7 @@ def test_complete_softmax_bjh_loop(self, isolated_workspace, mocker): # Mock LLMService to verify it's NOT called mock_llm = MagicMock() + mock_llm.use_mock = False # exercise the real grading path (fast-filter + LLM) with patch('engine.main.LLMService', return_value=mock_llm): result = runner.invoke(app, ["submit-justification", shallow_answer]) @@ -181,6 +197,7 @@ def test_complete_softmax_bjh_loop(self, isolated_workspace, mocker): # Mock LLM to return success mock_llm = MagicMock() + mock_llm.use_mock = False # exercise the real grading path (fast-filter + LLM) mock_evaluation = LLMEvaluationResponse( is_correct=True, feedback="Excellent! You've demonstrated deep understanding of the numerical stability mechanism." @@ -269,18 +286,21 @@ def test_validation_chain_branches_correctly(self, isolated_workspace, mocker): # Test 1: Keyword "stability" should trigger fast filter mock_llm = MagicMock() + mock_llm.use_mock = False # exercise the real grading path (fast-filter + LLM) with patch('engine.main.LLMService', return_value=mock_llm): result = runner.invoke(app, ["submit-justification", "It improves stability"]) mock_llm.evaluate_justification.assert_not_called() # Test 2: Keyword "overflow" alone should trigger fast filter mock_llm = MagicMock() + mock_llm.use_mock = False # exercise the real grading path (fast-filter + LLM) with patch('engine.main.LLMService', return_value=mock_llm): result = runner.invoke(app, ["submit-justification", "It prevents overflow"]) mock_llm.evaluate_justification.assert_not_called() # Test 3: No matching keywords should call LLM mock_llm = MagicMock() + mock_llm.use_mock = False # exercise the real grading path (fast-filter + LLM) mock_llm.evaluate_justification.return_value = LLMEvaluationResponse( is_correct=False, feedback="Try explaining the mathematical equivalence." @@ -288,3 +308,22 @@ def test_validation_chain_branches_correctly(self, isolated_workspace, mocker): with patch('engine.main.LLMService', return_value=mock_llm): result = runner.invoke(app, ["submit-justification", "The technique adjusts values"]) mock_llm.evaluate_justification.assert_called_once() + + # Test 4: No matching keywords + LLM ACCEPTS should advance past justify. + # (Covers the accept->advance path through the real command, complementing + # the unit-level coverage in test_submit_handlers.py.) + state_file = isolated_workspace["state_file"] + assert load_state(state_file).current_stage == "justify" # not advanced by Tests 1-3 + mock_llm = MagicMock() + mock_llm.use_mock = False + mock_llm.evaluate_justification.return_value = LLMEvaluationResponse( + is_correct=True, + feedback="Correct — the exp(c) factors cancel in the softmax ratio.", + ) + with patch('engine.main.LLMService', return_value=mock_llm): + result = runner.invoke( + app, + ["submit-justification", "Subtracting the max rescales logits; the shared factor cancels in the ratio."], + ) + mock_llm.evaluate_justification.assert_called_once() + assert load_state(state_file).current_stage == "harden", "accepted justification must advance to harden" diff --git a/tests/engine/test_bug_defs_match.py b/tests/engine/test_bug_defs_match.py new file mode 100644 index 0000000000..1c2527daad --- /dev/null +++ b/tests/engine/test_bug_defs_match.py @@ -0,0 +1,104 @@ +"""Phase-0 meta-test: every active cs336 Harden bug definition must inject cleanly. + +This is the regression guard for the Harden stage. For each active bug definition +(canonical defs; drafts/v2 are excluded), it: + + 1. resolves the reference source that defines the def's `target_function` + (developer reference preferred, student build-target as fallback), + 2. runs GenericBugInjector and asserts the injection SUCCEEDS (target found AND a + node matched — no NO_TARGET_FN / PATTERN_MISS), and + 3. asserts a matching `_symptom.txt` exists (required by HardenRunner._select_bug). + +If this test fails, the Harden stage is broken for that module: `mastery start-challenge` +would either crash or silently fail to inject a bug. +""" +from __future__ import annotations + +import ast +import json +from pathlib import Path + +import pytest + +from engine.ast_harden.generic_injector import GenericBugInjector + +REPO = Path(__file__).resolve().parents[2] +DEV = REPO / "modes" / "developer" / "cs336_basics" +STU = REPO / "modes" / "student" / "cs336_basics" +MODULES = REPO / "curricula" / "cs336_a1" / "modules" + + +def _reference_files() -> list[Path]: + ordered = sorted(DEV.glob("*.py")) + for p in sorted(STU.glob("*.py")): + if not (DEV / p.name).exists(): + ordered.append(p) + return ordered + + +def _defines(src: str, name: str) -> bool: + try: + tree = ast.parse(src) + except SyntaxError: + return False + return any( + isinstance(n, (ast.FunctionDef, ast.AsyncFunctionDef)) and n.name == name + for n in ast.walk(tree) + ) + + +def _resolve_reference(target: str) -> tuple[Path | None, str | None]: + for p in _reference_files(): + src = p.read_text(encoding="utf-8") + if _defines(src, target): + return p, src + return None, None + + +def _is_active(path: Path) -> bool: + stem = path.stem + return not (stem.endswith("_v2") or "_draft" in stem) + + +def _active_bug_defs() -> list[Path]: + return [p for p in sorted(MODULES.glob("*/bugs/*.json")) if _is_active(p)] + + +_ACTIVE = _active_bug_defs() + + +def test_active_bug_defs_discovered(): + """Guard against a glob/path regression silently collecting zero defs.""" + assert len(_ACTIVE) >= 19, f"expected >=19 active bug defs, found {len(_ACTIVE)}" + + +@pytest.mark.parametrize("bug_path", _ACTIVE, ids=lambda p: p.parent.parent.name) +def test_bug_def_injects(bug_path: Path): + bug = json.loads(bug_path.read_text(encoding="utf-8")) + target = bug.get("target_function") + assert target, f"{bug_path} has no target_function" + + ref_file, src = _resolve_reference(target) + assert ref_file is not None, ( + f"NO_TARGET_FN: '{target}' is not defined in any reference file " + f"(module {bug_path.parent.parent.name})" + ) + + buggy, ok = GenericBugInjector(bug).inject(src) + assert ok, ( + f"PATTERN_MISS: target '{target}' found in {ref_file.name} but no node matched " + f"(module {bug_path.parent.parent.name})" + ) + # A successful injection must actually change the code. + assert ast.dump(ast.parse(buggy)) != ast.dump(ast.parse(src)), ( + f"injection reported success but produced identical code ({bug_path.parent.parent.name})" + ) + + +@pytest.mark.parametrize("bug_path", _ACTIVE, ids=lambda p: p.parent.parent.name) +def test_bug_def_has_symptom(bug_path: Path): + symptom = bug_path.parent / f"{bug_path.stem}_symptom.txt" + assert symptom.exists(), ( + f"HardenRunner._select_bug requires {symptom.name} next to {bug_path.name}" + ) + assert symptom.read_text(encoding="utf-8").strip(), f"{symptom.name} is empty" diff --git a/tests/engine/test_bug_effectiveness.py b/tests/engine/test_bug_effectiveness.py new file mode 100644 index 0000000000..8323a13579 --- /dev/null +++ b/tests/engine/test_bug_effectiveness.py @@ -0,0 +1,162 @@ +"""Phase-0 EFFECTIVENESS guard: every Harden bug must actually fail its module's test. + +`test_bug_defs_match.py` proves each bug *injects and changes code*. That is necessary +but not sufficient: a bug that changes code the test doesn't exercise is a "silent bug" +(the design doc's worst case — it hands the learner an unsolvable/undetectable task). + +This test closes that gap empirically. For every active cs336 bug definition it: + + 1. builds a throwaway sandbox = developer reference `cs336_basics` + the assignment + test suite + fixtures (a faithful copy of what the validator runs against), + 2. runs the module's REAL pytest node against the CLEAN reference and asserts it PASSES + (baseline — proves the node/setup is valid), then + 3. injects the bug, runs the SAME node, and asserts it now FAILS (the bug is detected). + +It is non-destructive (operates entirely in a temp dir) and marked `slow` because each +case shells out to a real torch-backed pytest run. Deselect with `-m "not slow"`. + +The (module -> real pytest node) map is the source of truth the curriculum validators +should agree with; a mismatch here means a validator points at the wrong/te missing test. +""" +from __future__ import annotations + +import ast +import json +import os +import shutil +import subprocess +import sys +from pathlib import Path + +import pytest + +from engine.ast_harden.generic_injector import GenericBugInjector + +pytestmark = pytest.mark.slow + +REPO = Path(__file__).resolve().parents[2] +DEV = REPO / "modes" / "developer" / "cs336_basics" +TESTS = REPO / "tests" +MODULES = REPO / "curricula" / "cs336_a1" / "modules" + +# module -> the REAL pytest node that exercises that module's reference function. +# Kept explicit (not derived) so a drifting curriculum validator is caught by comparison. +NODE = { + "softmax": "test_nn_utils.py::test_softmax_matches_pytorch", + "cross_entropy": "test_nn_utils.py::test_cross_entropy", + "gradient_clipping": "test_nn_utils.py::test_gradient_clipping", + "linear": "test_model.py::test_linear", + "embedding": "test_model.py::test_embedding", + "silu": "test_model.py::test_silu_matches_pytorch", + "rmsnorm": "test_model.py::test_rmsnorm", + "swiglu": "test_model.py::test_swiglu", + "attention": "test_model.py::test_scaled_dot_product_attention", + "rope": "test_model.py::test_rope", + "multihead_attention": "test_model.py::test_multihead_self_attention_with_rope", + "transformer_block": "test_model.py::test_transformer_block", + "transformer_lm": "test_model.py::test_transformer_lm", + "adamw": "test_optimizer.py::test_adamw", + "cosine_schedule": "test_optimizer.py::test_get_lr_cosine_schedule", + "data_loader": "test_data.py::test_get_batch", + "checkpointing": "test_serialization.py::test_checkpointing", + "bpe_tokenizer": "test_train_bpe.py::test_train_bpe", + "tokenizer_class": "test_tokenizer.py::test_overlapping_special_tokens", +} + +# Assignment test assets to mirror into the sandbox (NOT tests/engine or tests/e2e, +# which would recurse into this very test). +_TEST_FILES = [ + "__init__.py", "adapters.py", "common.py", "conftest.py", "one_d_probes.py", + "test_data.py", "test_model.py", "test_nn_utils.py", "test_optimizer.py", + "test_serialization.py", "test_tokenizer.py", "test_train_bpe.py", +] +_TEST_DIRS = ["_snapshots", "fixtures"] + + +def _defines(src: str, name: str) -> bool: + try: + tree = ast.parse(src) + except SyntaxError: + return False + return any( + isinstance(n, (ast.FunctionDef, ast.AsyncFunctionDef)) and n.name == name + for n in ast.walk(tree) + ) + + +def _reference_file_for(target: str) -> Path: + for p in sorted(DEV.glob("*.py")): + if _defines(p.read_text(encoding="utf-8"), target): + return p + raise AssertionError(f"no developer reference defines '{target}'") + + +@pytest.fixture(scope="module") +def sandbox(tmp_path_factory) -> Path: + """A faithful, throwaway copy of {developer cs336_basics + assignment tests}.""" + root = tmp_path_factory.mktemp("bug_effectiveness") + shutil.copytree(DEV, root / "cs336_basics") + tdir = root / "tests" + tdir.mkdir() + (tdir / "__init__.py").write_text("", encoding="utf-8") + for name in _TEST_FILES: + src = TESTS / name + if src.exists(): + shutil.copy2(src, tdir / name) + for name in _TEST_DIRS: + src = TESTS / name + if src.exists(): + shutil.copytree(src, tdir / name) + return root + + +def _run_node(sandbox: Path, node: str) -> tuple[int, str]: + env = os.environ.copy() + env["PYTHONPATH"] = str(sandbox) + os.pathsep + env.get("PYTHONPATH", "") + proc = subprocess.run( + [sys.executable, "-m", "pytest", f"tests/{node}", + "--import-mode=importlib", "-q", "-p", "no:cacheprovider", "--no-header"], + cwd=sandbox, env=env, capture_output=True, text=True, timeout=600, + ) + return proc.returncode, proc.stdout + proc.stderr + + +_CASES = sorted(NODE.items()) + + +def test_node_map_covers_all_active_modules(): + """Every active bug module must have a real pytest node mapped here.""" + active = { + p.parent.parent.name + for p in MODULES.glob("*/bugs/*.json") + if not (p.stem.endswith("_v2") or "_draft" in p.stem) + } + missing = active - set(NODE) + assert not missing, f"active modules with no effectiveness node mapped: {sorted(missing)}" + + +@pytest.mark.parametrize("module,node", _CASES, ids=[m for m, _ in _CASES]) +def test_bug_actually_fails_its_test(sandbox: Path, module: str, node: str): + bug_path = next((MODULES / module / "bugs").glob("*.json")) + bug = json.loads(bug_path.read_text(encoding="utf-8")) + ref_name = _reference_file_for(bug["target_function"]).name + target = sandbox / "cs336_basics" / ref_name + clean_src = (DEV / ref_name).read_text(encoding="utf-8") + + try: + # Baseline: clean reference must PASS (proves node + sandbox are valid). + target.write_text(clean_src, encoding="utf-8") + rc_clean, out_clean = _run_node(sandbox, node) + assert rc_clean == 0, f"clean reference failed {node} (setup invalid):\n{out_clean[-1500:]}" + + # Inject the bug; the SAME node must now FAIL (bug is not silent). + buggy_src, ok = GenericBugInjector(bug).inject(clean_src) + assert ok, f"bug failed to inject into {ref_name} for module '{module}'" + target.write_text(buggy_src, encoding="utf-8") + rc_bug, _ = _run_node(sandbox, node) + assert rc_bug != 0, ( + f"SILENT BUG: '{bug.get('id')}' injects but {node} still PASSES — " + f"the learner would get an undetectable/unsolvable Harden challenge." + ) + finally: + target.write_text(clean_src, encoding="utf-8") # always restore the sandbox file