From c9b1cb80169656dfe7cc51eea43866ddfc4d7322 Mon Sep 17 00:00:00 2001 From: James Sesler <124222483+TheAmericanMaker@users.noreply.github.com> Date: Sun, 5 Apr 2026 02:34:54 -0400 Subject: [PATCH 1/2] Fix audit findings and harden runtime paths --- README.md | 14 ++++-- src/broadside_ai/backends/openai.py | 5 ++- src/broadside_ai/cli.py | 19 +++++--- src/broadside_ai/run.py | 31 +++++++++++-- src/broadside_ai/scatter.py | 10 +++-- src/broadside_ai/strategies/weighted_merge.py | 13 +++++- tests/test_budget.py | 44 +++++++++++++++++++ tests/test_cli.py | 41 +++++++++++++++++ tests/test_integration.py | 9 +++- tests/test_weighted_merge.py | 18 ++++++++ 10 files changed, 184 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index a17db61..1bb6928 100644 --- a/README.md +++ b/README.md @@ -67,11 +67,12 @@ broadside-ai --help Use the module entrypoint instead: ```cmd -py -3 -m broadside_ai --help +py -3.11 -m broadside_ai --help ``` -This works even when the user Scripts directory is not on `PATH`. You can use -`py -3 -m broadside_ai` anywhere this README shows `broadside-ai`. +Use the same Python launcher version you installed Broadside-AI into. On +machines with multiple Python installs, `py -3` may point at a different +interpreter and fail to find the package. If you prefer the console-script form, install with `pipx` so the command is added to a CLI-friendly location. @@ -136,6 +137,11 @@ broadside-ai run --prompt "Write a pitch for a dotfile manager" --n 3 --model ge That should print one synthesized result to stdout. +Examples below that reference repository files such as `RELEASE.md`, +`tasks/...`, or `benchmarks/...` assume you are in a checkout of this repo. +A plain `pip install broadside-ai` does not place those files in your current +working directory, so use your own local files or create a task YAML first. + ### Plain CLI output `run` prints only the synthesized result to stdout by default, which makes it @@ -263,7 +269,7 @@ broadside_ai_output/{model}/{topic}_{timestamp}/ ### Validate task files ```bash -broadside-ai validate-task tasks/ticket_classification.yaml +broadside-ai validate-task my_task.yaml ``` Validation exits `0` when every file is valid and `1` when any file fails. diff --git a/src/broadside_ai/backends/openai.py b/src/broadside_ai/backends/openai.py index 78d23fb..80438e5 100644 --- a/src/broadside_ai/backends/openai.py +++ b/src/broadside_ai/backends/openai.py @@ -44,8 +44,9 @@ def __init__( self.model = model self.max_tokens = max_tokens client_kwargs: dict[str, Any] = {"api_key": resolved_key} - if base_url: - client_kwargs["base_url"] = base_url + resolved_base_url = base_url or os.environ.get("OPENAI_BASE_URL") + if resolved_base_url: + client_kwargs["base_url"] = resolved_base_url self._client = openai.AsyncOpenAI(**client_kwargs) async def complete(self, prompt: str, **kwargs: Any) -> AgentResult: diff --git a/src/broadside_ai/cli.py b/src/broadside_ai/cli.py index c0e5ed4..e820baa 100644 --- a/src/broadside_ai/cli.py +++ b/src/broadside_ai/cli.py @@ -129,9 +129,13 @@ def run( if not task_file and not prompt: raise click.UsageError("Provide a task file or --prompt.") - task = _load_task_file(task_file) if task_file else Task(prompt=prompt or "") - if context_files: - task = _merge_task_context(task, _load_context_files(context_files)) + try: + task = _load_task_file(task_file) if task_file else Task(prompt=prompt or "") + if context_files: + task = _merge_task_context(task, _load_context_files(context_files)) + except (OSError, TypeError, ValueError, json.JSONDecodeError, yaml.YAMLError) as exc: + raise click.ClickException(str(exc)) from exc + backend_kwargs: dict[str, Any] = {} if model: backend_kwargs["model"] = model @@ -375,7 +379,7 @@ def _truncate_prompt(prompt: str, max_len: int = 60) -> str: def _load_task_file(path: str) -> Task: file_path = Path(path) - text = file_path.read_text() + text = file_path.read_text(encoding="utf-8") if file_path.suffix in (".yaml", ".yml"): data = yaml.safe_load(text) elif file_path.suffix == ".json": @@ -383,6 +387,11 @@ def _load_task_file(path: str) -> Task: else: return Task(prompt=text.strip()) + if not isinstance(data, dict): + raise ValueError( + f"Task file '{path}' must contain a YAML or JSON object with a 'prompt' field." + ) + task_data = {key: value for key, value in data.items() if key != "meta"} return Task(**task_data) @@ -402,7 +411,7 @@ def _load_context_files(paths: tuple[str, ...]) -> dict[str, str]: path = Path(raw_path) key = _context_key(path, used_keys) used_keys.add(key) - context[key] = path.read_text(encoding="utf-8").strip() + context[key] = path.read_text(encoding="utf-8") return context diff --git a/src/broadside_ai/run.py b/src/broadside_ai/run.py index 46cc93a..ce9e6b0 100644 --- a/src/broadside_ai/run.py +++ b/src/broadside_ai/run.py @@ -2,6 +2,7 @@ from __future__ import annotations +import threading import time from typing import Any @@ -76,8 +77,8 @@ def run_sync( """Synchronous version of run() for scripts and notebooks.""" import asyncio - return asyncio.run( - run( + async def _run_async() -> Synthesis: + return await run( task=task, n=n, backend=backend, @@ -90,4 +91,28 @@ def run_sync( parallel=parallel, early_stop=early_stop, ) - ) + + try: + asyncio.get_running_loop() + except RuntimeError: + return asyncio.run(_run_async()) + + result: Synthesis | None = None + error: BaseException | None = None + + def _runner() -> None: + nonlocal result, error + try: + result = asyncio.run(_run_async()) + except BaseException as exc: # pragma: no cover - exercised via re-raise below + error = exc + + thread = threading.Thread(target=_runner, daemon=True) + thread.start() + thread.join() + + if error is not None: + raise error + + assert result is not None + return result diff --git a/src/broadside_ai/scatter.py b/src/broadside_ai/scatter.py index b11bfb3..adefe6e 100644 --- a/src/broadside_ai/scatter.py +++ b/src/broadside_ai/scatter.py @@ -51,10 +51,14 @@ async def _run_one(index: int) -> AgentResult | None: return None try: result = await llm.complete(prompt, **kwargs) - budget_tracker.record(result.total_tokens) + try: + budget_tracker.record(result.total_tokens) + except BudgetExceeded: + logger.debug( + "Scatter budget exhausted after branch %d; keeping the completed result", + index, + ) return result - except BudgetExceeded: - return None except Exception as exc: logger.debug("Scatter branch %d failed: %s", index, exc) last_error = exc diff --git a/src/broadside_ai/strategies/weighted_merge.py b/src/broadside_ai/strategies/weighted_merge.py index 5337fe4..1c509a2 100644 --- a/src/broadside_ai/strategies/weighted_merge.py +++ b/src/broadside_ai/strategies/weighted_merge.py @@ -70,7 +70,12 @@ def _merge_fields(outputs: list[dict[str, Any]], weights: list[float]) -> dict[s raw_values = [value for value, _ in values] value_weights = [weight for _, weight in values] - if all(isinstance(value, (int, float)) for value in raw_values): + if all(isinstance(value, bool) for value in raw_values): + merged[key] = _merge_bool(raw_values) + elif all( + isinstance(value, (int, float)) and not isinstance(value, bool) + for value in raw_values + ): merged[key] = _merge_numeric(raw_values, value_weights) elif all(isinstance(value, str) for value in raw_values): merged[key] = _merge_categorical([str(value) for value in raw_values]) @@ -98,6 +103,10 @@ def _merge_categorical(values: list[str]) -> str: return Counter(values).most_common(1)[0][0] +def _merge_bool(values: list[bool]) -> bool: + return values.count(True) > (len(values) / 2) + + def _merge_lists(values: list[list[Any]]) -> list[Any]: from collections import Counter @@ -105,7 +114,7 @@ def _merge_lists(values: list[list[Any]]) -> list[Any]: for value_list in values: all_items.extend(value_list) - threshold = len(values) / 2 + threshold = (len(values) // 2) + 1 counts = Counter(str(item) for item in all_items) seen: set[str] = set() result: list[Any] = [] diff --git a/tests/test_budget.py b/tests/test_budget.py index e9e292f..afc5cdc 100644 --- a/tests/test_budget.py +++ b/tests/test_budget.py @@ -2,7 +2,34 @@ import pytest +from broadside_ai.backends import register +from broadside_ai.backends.base import AgentResult, Backend from broadside_ai.budget import BudgetExceeded, ScatterBudget +from broadside_ai.scatter import scatter +from broadside_ai.task import Task + + +class BudgetProbeBackend(Backend): + """Backend with configurable token usage for budget tests.""" + + def __init__(self, tokens_per_side: int = 500, **kwargs: object) -> None: + self.tokens_per_side = tokens_per_side + + async def complete(self, prompt: str, **kwargs: object) -> AgentResult: + return AgentResult( + text="budget probe", + tokens_in=self.tokens_per_side, + tokens_out=self.tokens_per_side, + latency_ms=5.0, + model="budget-probe", + backend="budget-probe", + ) + + def name(self) -> str: + return "budget-probe" + + +register("budget-probe", BudgetProbeBackend) def test_unbounded_budget(): @@ -46,3 +73,20 @@ def test_budget_exact_boundary_raises(): with pytest.raises(BudgetExceeded): b.record(1000) assert b.exhausted + + +@pytest.mark.asyncio +async def test_scatter_keeps_result_that_exhausts_budget(): + budget = ScatterBudget(max_tokens=1000) + + results = await scatter( + Task(prompt="test"), + n=3, + backend="budget-probe", + backend_kwargs={"tokens_per_side": 500}, + budget=budget, + parallel=False, + ) + + assert len(results) == 1 + assert results[0].text == "budget probe" diff --git a/tests/test_cli.py b/tests/test_cli.py index 33f5b6d..e08898d 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -114,6 +114,47 @@ def test_run_context_file_injects_local_file_into_prompt(): assert "Step two" in result.output +def test_run_context_file_preserves_significant_whitespace(): + runner = CliRunner() + with _isolated_fs(): + Path("context.txt").write_text( + " keep leading space\ntrailing newline\n", + encoding="utf-8", + ) + + result = runner.invoke( + main, + [ + "run", + "--prompt", + "Inspect spacing", + "--backend", + "echo-prompt", + "--raw", + "--n", + "1", + "--context-file", + "context.txt", + ], + ) + + assert result.exit_code == 0 + assert "context_txt:" in result.output + assert " keep leading space\ntrailing newline\n" in result.output + + +def test_run_with_non_mapping_task_file_reports_user_facing_error(): + runner = CliRunner() + with _isolated_fs(): + Path("bad.yaml").write_text("- nope", encoding="utf-8") + + result = runner.invoke(main, ["run", "bad.yaml", "--backend", "mock"]) + + assert result.exit_code == 1 + assert "Task file 'bad.yaml' must contain a YAML or JSON object" in result.output + assert "AttributeError" not in result.output + + def test_validate_task_command(): runner = CliRunner() with _isolated_fs(): diff --git a/tests/test_integration.py b/tests/test_integration.py index 9faaf2d..cab3471 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -3,7 +3,7 @@ import pytest from broadside_ai.gather import gather -from broadside_ai.run import run +from broadside_ai.run import run, run_sync from broadside_ai.scatter import scatter from broadside_ai.synthesize import synthesize from broadside_ai.task import Task @@ -39,3 +39,10 @@ async def test_full_pipeline_weighted_merge(): assert synthesis_result.requested_strategy == "weighted_merge" assert synthesis_result.parsed_result is not None assert synthesis_result.parsed_result["label"] == "spam" + + +@pytest.mark.asyncio +async def test_run_sync_works_inside_running_event_loop(): + result = run_sync(Task(prompt="What is 2+2?"), n=1, backend="mock") + assert result.strategy == "llm" + assert result.result diff --git a/tests/test_weighted_merge.py b/tests/test_weighted_merge.py index cacb7a6..e62a8e3 100644 --- a/tests/test_weighted_merge.py +++ b/tests/test_weighted_merge.py @@ -9,6 +9,7 @@ from broadside_ai.strategies.weighted_merge import ( _extract_weights, _merge_fields, + _merge_lists, synthesize_weighted_merge, ) @@ -109,3 +110,20 @@ def test_merge_fields_numeric_and_majority(): ) assert merged["score"] == pytest.approx(7.0, abs=0.01) assert merged["label"] == "good" + + +def test_merge_fields_booleans_use_majority_vote(): + merged = _merge_fields( + [ + {"approved": True}, + {"approved": False}, + {"approved": True}, + ], + [1 / 3, 1 / 3, 1 / 3], + ) + assert merged["approved"] is True + + +def test_merge_lists_requires_strict_majority(): + assert _merge_lists([["python"], ["ai"]]) == [] + assert _merge_lists([["python"], ["python"], ["ai"], ["ai"]]) == [] From f5e00e676b1dabea4353b34056a9ed84680560f2 Mon Sep 17 00:00:00 2001 From: James Sesler <124222483+TheAmericanMaker@users.noreply.github.com> Date: Sun, 5 Apr 2026 02:44:40 -0400 Subject: [PATCH 2/2] Apply Ruff formatting to weighted merge --- src/broadside_ai/strategies/weighted_merge.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/broadside_ai/strategies/weighted_merge.py b/src/broadside_ai/strategies/weighted_merge.py index 1c509a2..72628d1 100644 --- a/src/broadside_ai/strategies/weighted_merge.py +++ b/src/broadside_ai/strategies/weighted_merge.py @@ -73,8 +73,7 @@ def _merge_fields(outputs: list[dict[str, Any]], weights: list[float]) -> dict[s if all(isinstance(value, bool) for value in raw_values): merged[key] = _merge_bool(raw_values) elif all( - isinstance(value, (int, float)) and not isinstance(value, bool) - for value in raw_values + isinstance(value, (int, float)) and not isinstance(value, bool) for value in raw_values ): merged[key] = _merge_numeric(raw_values, value_weights) elif all(isinstance(value, str) for value in raw_values):