Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 10 additions & 4 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -67,11 +67,12 @@ broadside-ai --help
Use the module entrypoint instead:

```cmd
py -3 -m broadside_ai --help
py -3.11 -m broadside_ai --help
```

This works even when the user Scripts directory is not on `PATH`. You can use
`py -3 -m broadside_ai` anywhere this README shows `broadside-ai`.
Use the same Python launcher version you installed Broadside-AI into. On
machines with multiple Python installs, `py -3` may point at a different
interpreter and fail to find the package.

If you prefer the console-script form, install with `pipx` so the command is
added to a CLI-friendly location.
Expand Down Expand Up @@ -136,6 +137,11 @@ broadside-ai run --prompt "Write a pitch for a dotfile manager" --n 3 --model ge

That should print one synthesized result to stdout.

Examples below that reference repository files such as `RELEASE.md`,
`tasks/...`, or `benchmarks/...` assume you are in a checkout of this repo.
A plain `pip install broadside-ai` does not place those files in your current
working directory, so use your own local files or create a task YAML first.

### Plain CLI output

`run` prints only the synthesized result to stdout by default, which makes it
Expand Down Expand Up @@ -263,7 +269,7 @@ broadside_ai_output/{model}/{topic}_{timestamp}/
### Validate task files

```bash
broadside-ai validate-task tasks/ticket_classification.yaml
broadside-ai validate-task my_task.yaml
```

Validation exits `0` when every file is valid and `1` when any file fails.
Expand Down
5 changes: 3 additions & 2 deletions src/broadside_ai/backends/openai.py
Original file line number Diff line number Diff line change
Expand Up @@ -44,8 +44,9 @@ def __init__(
self.model = model
self.max_tokens = max_tokens
client_kwargs: dict[str, Any] = {"api_key": resolved_key}
if base_url:
client_kwargs["base_url"] = base_url
resolved_base_url = base_url or os.environ.get("OPENAI_BASE_URL")
if resolved_base_url:
client_kwargs["base_url"] = resolved_base_url
self._client = openai.AsyncOpenAI(**client_kwargs)

async def complete(self, prompt: str, **kwargs: Any) -> AgentResult:
Expand Down
19 changes: 14 additions & 5 deletions src/broadside_ai/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -129,9 +129,13 @@ def run(
if not task_file and not prompt:
raise click.UsageError("Provide a task file or --prompt.")

task = _load_task_file(task_file) if task_file else Task(prompt=prompt or "")
if context_files:
task = _merge_task_context(task, _load_context_files(context_files))
try:
task = _load_task_file(task_file) if task_file else Task(prompt=prompt or "")
if context_files:
task = _merge_task_context(task, _load_context_files(context_files))
except (OSError, TypeError, ValueError, json.JSONDecodeError, yaml.YAMLError) as exc:
raise click.ClickException(str(exc)) from exc

backend_kwargs: dict[str, Any] = {}
if model:
backend_kwargs["model"] = model
Expand Down Expand Up @@ -375,14 +379,19 @@ def _truncate_prompt(prompt: str, max_len: int = 60) -> str:

def _load_task_file(path: str) -> Task:
file_path = Path(path)
text = file_path.read_text()
text = file_path.read_text(encoding="utf-8")
if file_path.suffix in (".yaml", ".yml"):
data = yaml.safe_load(text)
elif file_path.suffix == ".json":
data = json.loads(text)
else:
return Task(prompt=text.strip())

if not isinstance(data, dict):
raise ValueError(
f"Task file '{path}' must contain a YAML or JSON object with a 'prompt' field."
)

task_data = {key: value for key, value in data.items() if key != "meta"}
return Task(**task_data)

Expand All @@ -402,7 +411,7 @@ def _load_context_files(paths: tuple[str, ...]) -> dict[str, str]:
path = Path(raw_path)
key = _context_key(path, used_keys)
used_keys.add(key)
context[key] = path.read_text(encoding="utf-8").strip()
context[key] = path.read_text(encoding="utf-8")
return context


Expand Down
31 changes: 28 additions & 3 deletions src/broadside_ai/run.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

from __future__ import annotations

import threading
import time
from typing import Any

Expand Down Expand Up @@ -76,8 +77,8 @@ def run_sync(
"""Synchronous version of run() for scripts and notebooks."""
import asyncio

return asyncio.run(
run(
async def _run_async() -> Synthesis:
return await run(
task=task,
n=n,
backend=backend,
Expand All @@ -90,4 +91,28 @@ def run_sync(
parallel=parallel,
early_stop=early_stop,
)
)

try:
asyncio.get_running_loop()
except RuntimeError:
return asyncio.run(_run_async())

result: Synthesis | None = None
error: BaseException | None = None

def _runner() -> None:
nonlocal result, error
try:
result = asyncio.run(_run_async())
except BaseException as exc: # pragma: no cover - exercised via re-raise below
error = exc

thread = threading.Thread(target=_runner, daemon=True)
thread.start()
thread.join()

if error is not None:
raise error

assert result is not None
return result
10 changes: 7 additions & 3 deletions src/broadside_ai/scatter.py
Original file line number Diff line number Diff line change
Expand Up @@ -51,10 +51,14 @@ async def _run_one(index: int) -> AgentResult | None:
return None
try:
result = await llm.complete(prompt, **kwargs)
budget_tracker.record(result.total_tokens)
try:
budget_tracker.record(result.total_tokens)
except BudgetExceeded:
logger.debug(
"Scatter budget exhausted after branch %d; keeping the completed result",
index,
)
return result
except BudgetExceeded:
return None
except Exception as exc:
logger.debug("Scatter branch %d failed: %s", index, exc)
last_error = exc
Expand Down
12 changes: 10 additions & 2 deletions src/broadside_ai/strategies/weighted_merge.py
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,11 @@ def _merge_fields(outputs: list[dict[str, Any]], weights: list[float]) -> dict[s
raw_values = [value for value, _ in values]
value_weights = [weight for _, weight in values]

if all(isinstance(value, (int, float)) for value in raw_values):
if all(isinstance(value, bool) for value in raw_values):
merged[key] = _merge_bool(raw_values)
elif all(
isinstance(value, (int, float)) and not isinstance(value, bool) for value in raw_values
):
merged[key] = _merge_numeric(raw_values, value_weights)
elif all(isinstance(value, str) for value in raw_values):
merged[key] = _merge_categorical([str(value) for value in raw_values])
Expand Down Expand Up @@ -98,14 +102,18 @@ def _merge_categorical(values: list[str]) -> str:
return Counter(values).most_common(1)[0][0]


def _merge_bool(values: list[bool]) -> bool:
return values.count(True) > (len(values) / 2)


def _merge_lists(values: list[list[Any]]) -> list[Any]:
from collections import Counter

all_items: list[Any] = []
for value_list in values:
all_items.extend(value_list)

threshold = len(values) / 2
threshold = (len(values) // 2) + 1
counts = Counter(str(item) for item in all_items)
seen: set[str] = set()
result: list[Any] = []
Expand Down
44 changes: 44 additions & 0 deletions tests/test_budget.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,34 @@

import pytest

from broadside_ai.backends import register
from broadside_ai.backends.base import AgentResult, Backend
from broadside_ai.budget import BudgetExceeded, ScatterBudget
from broadside_ai.scatter import scatter
from broadside_ai.task import Task


class BudgetProbeBackend(Backend):
"""Backend with configurable token usage for budget tests."""

def __init__(self, tokens_per_side: int = 500, **kwargs: object) -> None:
self.tokens_per_side = tokens_per_side

async def complete(self, prompt: str, **kwargs: object) -> AgentResult:
return AgentResult(
text="budget probe",
tokens_in=self.tokens_per_side,
tokens_out=self.tokens_per_side,
latency_ms=5.0,
model="budget-probe",
backend="budget-probe",
)

def name(self) -> str:
return "budget-probe"


register("budget-probe", BudgetProbeBackend)


def test_unbounded_budget():
Expand Down Expand Up @@ -46,3 +73,20 @@ def test_budget_exact_boundary_raises():
with pytest.raises(BudgetExceeded):
b.record(1000)
assert b.exhausted


@pytest.mark.asyncio
async def test_scatter_keeps_result_that_exhausts_budget():
budget = ScatterBudget(max_tokens=1000)

results = await scatter(
Task(prompt="test"),
n=3,
backend="budget-probe",
backend_kwargs={"tokens_per_side": 500},
budget=budget,
parallel=False,
)

assert len(results) == 1
assert results[0].text == "budget probe"
41 changes: 41 additions & 0 deletions tests/test_cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -114,6 +114,47 @@ def test_run_context_file_injects_local_file_into_prompt():
assert "Step two" in result.output


def test_run_context_file_preserves_significant_whitespace():
runner = CliRunner()
with _isolated_fs():
Path("context.txt").write_text(
" keep leading space\ntrailing newline\n",
encoding="utf-8",
)

result = runner.invoke(
main,
[
"run",
"--prompt",
"Inspect spacing",
"--backend",
"echo-prompt",
"--raw",
"--n",
"1",
"--context-file",
"context.txt",
],
)

assert result.exit_code == 0
assert "context_txt:" in result.output
assert " keep leading space\ntrailing newline\n" in result.output


def test_run_with_non_mapping_task_file_reports_user_facing_error():
runner = CliRunner()
with _isolated_fs():
Path("bad.yaml").write_text("- nope", encoding="utf-8")

result = runner.invoke(main, ["run", "bad.yaml", "--backend", "mock"])

assert result.exit_code == 1
assert "Task file 'bad.yaml' must contain a YAML or JSON object" in result.output
assert "AttributeError" not in result.output


def test_validate_task_command():
runner = CliRunner()
with _isolated_fs():
Expand Down
9 changes: 8 additions & 1 deletion tests/test_integration.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
import pytest

from broadside_ai.gather import gather
from broadside_ai.run import run
from broadside_ai.run import run, run_sync
from broadside_ai.scatter import scatter
from broadside_ai.synthesize import synthesize
from broadside_ai.task import Task
Expand Down Expand Up @@ -39,3 +39,10 @@ async def test_full_pipeline_weighted_merge():
assert synthesis_result.requested_strategy == "weighted_merge"
assert synthesis_result.parsed_result is not None
assert synthesis_result.parsed_result["label"] == "spam"


@pytest.mark.asyncio
async def test_run_sync_works_inside_running_event_loop():
result = run_sync(Task(prompt="What is 2+2?"), n=1, backend="mock")
assert result.strategy == "llm"
assert result.result
18 changes: 18 additions & 0 deletions tests/test_weighted_merge.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
from broadside_ai.strategies.weighted_merge import (
_extract_weights,
_merge_fields,
_merge_lists,
synthesize_weighted_merge,
)

Expand Down Expand Up @@ -109,3 +110,20 @@ def test_merge_fields_numeric_and_majority():
)
assert merged["score"] == pytest.approx(7.0, abs=0.01)
assert merged["label"] == "good"


def test_merge_fields_booleans_use_majority_vote():
merged = _merge_fields(
[
{"approved": True},
{"approved": False},
{"approved": True},
],
[1 / 3, 1 / 3, 1 / 3],
)
assert merged["approved"] is True


def test_merge_lists_requires_strict_majority():
assert _merge_lists([["python"], ["ai"]]) == []
assert _merge_lists([["python"], ["python"], ["ai"], ["ai"]]) == []
Loading