Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion envs/textarena_env/server/environment.py
Original file line number Diff line number Diff line change
Expand Up @@ -147,7 +147,7 @@ def reset(
if hasattr(env, "full_observations"):
env.full_observations = {}

self._ta_env.reset(num_players=self.num_players)
self._ta_env.reset(num_players=self.num_players, seed=seed)

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This call mutates process-global RNG state in TextArena 0.7.4 (SinglePlayerState.__init__State.__init__random.seed(seed)). A concurrent or subsequent unseeded session therefore observes the seeded stream. Serialize all underlying resets with a process-wide lock and save/restore RNG state for seeded calls; add coverage that proves seeded resets neither perturb nor race an unseeded session.


for provider in self._reward_providers:
provider.reset()
Expand Down
51 changes: 51 additions & 0 deletions tests/envs/test_textarena_environment.py
Original file line number Diff line number Diff line change
Expand Up @@ -65,3 +65,54 @@ def test_wordle_reset_clears_accumulated_state():
# Verify the prompts are actually the same content
assert obs1.prompt == obs2.prompt
assert obs2.prompt == obs3.prompt


def _secret_word(env: TextArenaEnvironment) -> str:
"""Read the puzzle from TextArena's own game state, not from the prompt.

Wordle's prompt is static instructions, so hashing the observation would
report every episode as identical whether or not the seed took effect.
"""
return env._ta_env.state.game_state["secret_word"]


def test_reset_seed_is_forwarded_to_textarena():
"""A fixed seed must give the same episode, and different seeds different ones."""
pytest.importorskip("textarena", reason="textarena not installed")
env = TextArenaEnvironment(env_id="Wordle-v0", num_players=1)

seeded = []
for _ in range(3):
env.reset(seed=1234)
seeded.append(_secret_word(env))

env.reset(seed=999)
other_seed = _secret_word(env)

assert len(set(seeded)) == 1, f"seed=1234 produced {sorted(set(seeded))}"
assert other_seed != seeded[0], "a different seed produced the same episode"


def test_reset_seed_is_reproducible_across_instances():
"""The same seed must survive constructing a fresh environment."""
pytest.importorskip("textarena", reason="textarena not installed")
first = TextArenaEnvironment(env_id="Wordle-v0", num_players=1)
first.reset(seed=1234)

second = TextArenaEnvironment(env_id="Wordle-v0", num_players=1)
second.reset(seed=1234)

assert _secret_word(first) == _secret_word(second)


def test_reset_without_seed_still_varies():
"""Forwarding the seed must not accidentally pin unseeded episodes."""
pytest.importorskip("textarena", reason="textarena not installed")
env = TextArenaEnvironment(env_id="Wordle-v0", num_players=1)

words = set()
for _ in range(8):
env.reset()
words.add(_secret_word(env))

assert len(words) > 1, "unseeded resets should not be deterministic"