merge main (again)

.github/workflows/build.yml

@@ -9,12 +9,14 @@ on:
       - 'setup.py'
       - 'requirements.txt'
       - '*.iss'
+      - 'worlds/*/archipelago.json'
   pull_request:
     paths:
       - '.github/workflows/build.yml'
       - 'setup.py'
       - 'requirements.txt'
       - '*.iss'
+      - 'worlds/*/archipelago.json'
   workflow_dispatch:
 
 env:

CommonClient.py

@@ -856,9 +856,9 @@ async def server_loop(ctx: CommonContext, address: typing.Optional[str] = None)
 
     server_url = urllib.parse.urlparse(address)
     if server_url.username:
-        ctx.username = server_url.username
+        ctx.username = urllib.parse.unquote(server_url.username)
     if server_url.password:
-        ctx.password = server_url.password
+        ctx.password = urllib.parse.unquote(server_url.password)
 
     def reconnect_hint() -> str:
         return ", type /connect to reconnect" if ctx.server_address else ""

Generate.py

@@ -23,7 +23,7 @@
 from Utils import parse_yamls, version_tuple, __version__, tuplize_version
 
 
-def mystery_argparse():
+def mystery_argparse(argv: list[str] | None = None):
     from settings import get_settings
     settings = get_settings()
     defaults = settings.generator
@@ -57,7 +57,7 @@ def mystery_argparse():
     parser.add_argument("--spoiler_only", action="store_true",
                         help="Skips generation assertion and multidata, outputting only a spoiler log. "
                              "Intended for debugging and testing purposes.")
-    args = parser.parse_args()
+    args = parser.parse_args(argv)
 
     if args.skip_output and args.spoiler_only:
         parser.error("Cannot mix --skip_output and --spoiler_only")

MultiServer.py

@@ -135,6 +135,7 @@ def get_saving_second(seed_name: str, interval: int = 60) -> int:
 
 class Client(Endpoint):
     __slots__ = (
+        "__weakref__",
         "version",
         "auth",
         "team",
@@ -1199,16 +1200,17 @@ def collect_hints(ctx: Context, team: int, slot: int, item: typing.Union[int, st
             found = location_id in ctx.location_checks[team, finding_player]
             entrance = ctx.er_hint_data.get(finding_player, {}).get(location_id, "")
 
+            hint_status = status  # Assign again because we're in a for loop
             if found:
-                status = HintStatus.HINT_FOUND
-            elif status is None:
+                hint_status = HintStatus.HINT_FOUND
+            elif hint_status is None:
                 if item_flags & ItemClassification.trap:
-                    status = HintStatus.HINT_AVOID
+                    hint_status = HintStatus.HINT_AVOID
                 else:
-                    status = HintStatus.HINT_PRIORITY
+                    hint_status = HintStatus.HINT_PRIORITY
 
             hints.append(
-                Hint(receiving_player, finding_player, location_id, item_id, found, entrance, item_flags, status)
+                Hint(receiving_player, finding_player, location_id, item_id, found, entrance, item_flags, hint_status)
             )
 
     return hints

Options.py

@@ -1474,8 +1474,10 @@ def verify(self, world: typing.Type[World], player_name: str, plando_options: "P
         super(ItemLinks, self).verify(world, player_name, plando_options)
         existing_links = set()
         for link in self.value:
+            link["name"] = link["name"].strip()[:16].strip()
             if link["name"] in existing_links:
-                raise Exception(f"You cannot have more than one link named {link['name']}.")
+                raise Exception(f"Item link names are limited to their first 16 characters and must be unique. "
+                                f"You have more than one link named '{link['name']}'.")
             existing_links.add(link["name"])
 
             pool = self.verify_items(link["item_pool"], link["name"], "item_pool", world)

Utils.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import asyncio
+import concurrent.futures
 import json
 import typing
 import builtins
@@ -477,7 +478,7 @@ def find_class(self, module: str, name: str) -> type:
                 mod = importlib.import_module(module)
             obj = getattr(mod, name)
             if issubclass(obj, (self.options_module.Option, self.options_module.PlandoConnection,
-                                self.options_module.PlandoText)):
+                                self.options_module.PlandoItem, self.options_module.PlandoText)):
                 return obj
         # Forbid everything else.
         raise pickle.UnpicklingError(f"global '{module}.{name}' is forbidden")
@@ -1138,3 +1139,40 @@ def is_iterable_except_str(obj: object) -> TypeGuard[typing.Iterable[typing.Any]
     if isinstance(obj, str):
         return False
     return isinstance(obj, typing.Iterable)
+
+
+class DaemonThreadPoolExecutor(concurrent.futures.ThreadPoolExecutor):
+    """
+    ThreadPoolExecutor that uses daemonic threads that do not keep the program alive.
+    NOTE: use this with caution because killed threads will not properly clean up.
+    """
+
+    def _adjust_thread_count(self):
+        # see upstream ThreadPoolExecutor for details
+        import threading
+        import weakref
+        from concurrent.futures.thread import _worker
+
+        if self._idle_semaphore.acquire(timeout=0):
+            return
+
+        def weakref_cb(_, q=self._work_queue):
+            q.put(None)
+
+        num_threads = len(self._threads)
+        if num_threads < self._max_workers:
+            thread_name = f"{self._thread_name_prefix or self}_{num_threads}"
+            t = threading.Thread(
+                name=thread_name,
+                target=_worker,
+                args=(
+                    weakref.ref(self, weakref_cb),
+                    self._work_queue,
+                    self._initializer,
+                    self._initargs,
+                ),
+                daemon=True,
+            )
+            t.start()
+            self._threads.add(t)
+            # NOTE: don't add to _threads_queues so we don't block on shutdown

WebHostLib/__init__.py

@@ -1,6 +1,7 @@
 import base64
 import os
 import socket
+import typing
 import uuid
 
 from flask import Flask
@@ -61,20 +62,21 @@
 Compress(app)
 
 
-def to_python(value):
+def to_python(value: str) -> uuid.UUID:
     return uuid.UUID(bytes=base64.urlsafe_b64decode(value + '=='))
 
 
-def to_url(value):
+def to_url(value: uuid.UUID) -> str:
     return base64.urlsafe_b64encode(value.bytes).rstrip(b'=').decode('ascii')
 
 
 class B64UUIDConverter(BaseConverter):
 
-    def to_python(self, value):
+    def to_python(self, value: str) -> uuid.UUID:
         return to_python(value)
 
-    def to_url(self, value):
+    def to_url(self, value: typing.Any) -> str:
+        assert isinstance(value, uuid.UUID)
         return to_url(value)
 
 
@@ -84,7 +86,7 @@ def to_url(self, value):
 app.jinja_env.filters["title_sorted"] = title_sorted
 
 
-def register():
+def register() -> None:
     """Import submodules, triggering their registering on flask routing.
     Note: initializes worlds subsystem."""
     import importlib

WebHostLib/autolauncher.py

@@ -17,7 +17,7 @@
 _stop_event = Event()
 
 
-def stop():
+def stop() -> None:
     """Stops previously launched threads"""
     global _stop_event
     stop_event = _stop_event
@@ -36,25 +36,39 @@ def handle_generation_failure(result: BaseException):
         logging.exception(e)
 
 
-def _mp_gen_game(gen_options: dict, meta: dict[str, Any] | None = None, owner=None, sid=None) -> PrimaryKey | None:
+def _mp_gen_game(
+    gen_options: dict,
+    meta: dict[str, Any] | None = None,
+    owner=None,
+    sid=None,
+    timeout: int|None = None,
+) -> PrimaryKey | None:
     from setproctitle import setproctitle
 
     setproctitle(f"Generator ({sid})")
-    res = gen_game(gen_options, meta=meta, owner=owner, sid=sid)
-    setproctitle(f"Generator (idle)")
-    return res
+    try:
+        return gen_game(gen_options, meta=meta, owner=owner, sid=sid, timeout=timeout)
+    finally:
+        setproctitle(f"Generator (idle)")
 
 
-def launch_generator(pool: multiprocessing.pool.Pool, generation: Generation):
+def launch_generator(pool: multiprocessing.pool.Pool, generation: Generation, timeout: int|None) -> None:
     try:
         meta = json.loads(generation.meta)
         options = restricted_loads(generation.options)
         logging.info(f"Generating {generation.id} for {len(options)} players")
-        pool.apply_async(_mp_gen_game, (options,),
-                         {"meta": meta,
-                          "sid": generation.id,
-                          "owner": generation.owner},
-                         handle_generation_success, handle_generation_failure)
+        pool.apply_async(
+            _mp_gen_game,
+            (options,),
+            {
+                "meta": meta,
+                "sid": generation.id,
+                "owner": generation.owner,
+                "timeout": timeout,
+            },
+            handle_generation_success,
+            handle_generation_failure,
+        )
     except Exception as e:
         generation.state = STATE_ERROR
         commit()
@@ -135,6 +149,7 @@ def keep_running():
 
                 with multiprocessing.Pool(config["GENERATORS"], initializer=init_generator,
                                           initargs=(config,), maxtasksperchild=10) as generator_pool:
+                    job_time = config["JOB_TIME"]
                     with db_session:
                         to_start = select(generation for generation in Generation if generation.state == STATE_STARTED)
 
@@ -145,7 +160,7 @@ def keep_running():
                                 if sid:
                                     generation.delete()
                                 else:
-                                    launch_generator(generator_pool, generation)
+                                    launch_generator(generator_pool, generation, timeout=job_time)
 
                             commit()
                         select(generation for generation in Generation if generation.state == STATE_ERROR).delete()
@@ -157,7 +172,7 @@ def keep_running():
                                 generation for generation in Generation
                                 if generation.state == STATE_QUEUED).for_update()
                             for generation in to_start:
-                                launch_generator(generator_pool, generation)
+                                launch_generator(generator_pool, generation, timeout=job_time)
         except AlreadyRunningException:
             logging.info("Autogen reports as already running, not starting another.")
 

WebHostLib/generate.py

@@ -14,7 +14,7 @@
 from BaseClasses import get_seed, seeddigits
 from Generate import PlandoOptions, handle_name, mystery_argparse
 from Main import main as ERmain
-from Utils import __version__, restricted_dumps
+from Utils import __version__, restricted_dumps, DaemonThreadPoolExecutor
 from WebHostLib import app
 from settings import ServerOptions, GeneratorOptions
 from .check import get_yaml_data, roll_options
@@ -107,7 +107,7 @@ def start_generation(options: dict[str, dict | str], meta: dict[str, Any]):
     else:
         try:
             seed_id = gen_game({name: vars(options) for name, options in gen_options.items()},
-                               meta=meta, owner=session["_id"].int)
+                               meta=meta, owner=session["_id"].int, timeout=app.config["JOB_TIME"])
         except BaseException as e:
             from .autolauncher import handle_generation_failure
             handle_generation_failure(e)
@@ -118,7 +118,7 @@ def start_generation(options: dict[str, dict | str], meta: dict[str, Any]):
         return redirect(url_for("view_seed", seed=seed_id))
 
 
-def gen_game(gen_options: dict, meta: dict[str, Any] | None = None, owner=None, sid=None):
+def gen_game(gen_options: dict, meta: dict[str, Any] | None = None, owner=None, sid=None, timeout: int|None = None):
     if meta is None:
         meta = {}
 
@@ -137,7 +137,7 @@ def task():
 
         seedname = "W" + (f"{random.randint(0, pow(10, seeddigits) - 1)}".zfill(seeddigits))
 
-        args = mystery_argparse()
+        args = mystery_argparse([])  # Just to set up the Namespace with defaults
         args.multi = playercount
         args.seed = seed
         args.name = {x: "" for x in range(1, playercount + 1)}  # only so it can be overwritten in mystery
@@ -172,11 +172,12 @@ def task():
         ERmain(args, seed, baked_server_options=meta["server_options"])
 
         return upload_to_db(target.name, sid, owner, race)
-    thread_pool = concurrent.futures.ThreadPoolExecutor(max_workers=1)
+
+    thread_pool = DaemonThreadPoolExecutor(max_workers=1)
     thread = thread_pool.submit(task)
 
     try:
-        return thread.result(app.config["JOB_TIME"])
+        return thread.result(timeout)
     except concurrent.futures.TimeoutError as e:
         if sid:
             with db_session:
@@ -189,6 +190,9 @@ def task():
                                      format_exception(e))
                     gen.meta = json.dumps(meta)
                     commit()
+    except (KeyboardInterrupt, SystemExit):
+        # don't update db, retry next time
+        raise
     except BaseException as e:
         if sid:
             with db_session:
@@ -200,6 +204,11 @@ def task():
                     gen.meta = json.dumps(meta)
                     commit()
         raise
+    finally:
+        # free resources claimed by thread pool, if possible
+        # NOTE: Timeout depends on the process being killed at some point
+        #       since we can't actually cancel a running gen at the moment.
+        thread_pool.shutdown(wait=False, cancel_futures=True)
 
 
 @app.route('/wait/<suuid:seed>')

WebHostLib/options.py

@@ -76,7 +76,7 @@ def filter_rst_to_html(text: str) -> str:
         lines = text.splitlines()
         text = lines[0] + "\n" + dedent("\n".join(lines[1:]))
 
-    return publish_parts(text, writer_name='html', settings=None, settings_overrides={
+    return publish_parts(text, writer='html', settings=None, settings_overrides={
         'raw_enable': False,
         'file_insertion_enabled': False,
         'output_encoding': 'unicode'

WebHostLib/requirements.txt

@@ -4,7 +4,8 @@ pony>=0.7.19; python_version <= '3.12'
 pony @ git+https://github.com/black-sliver/pony@7feb1221953b7fa4a6735466bf21a8b4d35e33ba#0.7.19; python_version >= '3.13'
 waitress>=3.0.2
 Flask-Caching>=2.3.0
-Flask-Compress>=1.17
+Flask-Compress>=1.17; python_version >= '3.12'
+Flask-Compress==1.18; python_version <= '3.11'  # 3.11's pkg_resources can't resolve the new "backports.zstd" dependency
 Flask-Limiter>=3.12
 bokeh>=3.6.3
 markupsafe>=3.0.2