indexer: raise semantic worker clamp from 2 to 4 (#109)

app/config.py

@@ -93,12 +93,16 @@ class Settings(BaseSettings):
     # in-process memory rather than a DB lock; exceeding it fails loudly.
     #
     # Sized against the ACTUAL buffer cost, not a round number: the vectors are held as
-    # Python float lists, ~32 B per element (24 B float object + 8 B list pointer), so at
-    # dim=1024 each chunk costs ~32 KB -- 8000 chunks is ~260 MB of vectors plus ~16 MB of
-    # chunk text. A larger ceiling (e.g. 50k -> ~1.6 GB) would OOM the job container before
-    # this loud check could ever fire, which would defeat the point of having a ceiling.
-    # A repo that legitimately exceeds this needs a temp-table staging path, not a bigger
-    # buffer.
+    # Python float lists, ~32 B per element (24 B float object + 8 B list pointer) structural,
+    # but ~40.1 KB/chunk RESIDENT once measured (issue #109; pymalloc overhead/fragmentation --
+    # use this figure for headroom arithmetic) -- 8000 chunks is ~313 MiB of vectors resident
+    # plus ~16 MB of chunk text. #109 also derived a per-worker chunk-cap ceiling from a full
+    # container-memory model (~73,300 chunks at the pinned N=2 semantic-worker count this cap
+    # is evaluated at, ~36,700 at the shipped N=4 -- see docs/perf/issue-109-measurements.md
+    # §12): a larger ceiling well past that (e.g. 50k, ~1.9 GiB resident) risks OOMing the job
+    # container before this loud check could ever fire, which would defeat the point of having
+    # a ceiling. A repo that legitimately exceeds this needs a temp-table staging path, not a
+    # bigger buffer.
     #
     # Scope note (#104): under file-level delta indexing this cap is enforced against
     # whatever ONE RUN embeds (changed/new + membership-only files), not a branch's whole
@@ -112,11 +116,12 @@ class Settings(BaseSettings):
     # which bounds file ingestion, not embedding-chunk granularity.
     semantic_chunk_max_tokens: int = 512
 
-    # In-flight embedding requests per worker (#107). The indexer clamps to 2 workers when
-    # semantic is on (indexer/repo_config.py:effective_workers), so total in-flight gateway
-    # requests are workers x concurrency: 2 x 4 = 8 at this default, 2 x 8 = 16 at the
-    # config.yaml-enforced ceiling of 8 -- both under the SDK's 20-connection pool
-    # (pool_block=True, so exceeding it would silently serialize rather than error). Setting
+    # In-flight embedding requests per worker (#107). The indexer clamps to 4 workers when
+    # semantic is on (indexer/repo_config.py:effective_workers -- issue #109 raised this from
+    # 2), so total in-flight gateway requests are workers x concurrency: 4 x 4 = 16 at this
+    # default, 4 x 8 = 32 at the config.yaml-enforced ceiling of 8 -- the latter now EXCEEDS
+    # the SDK's 20-connection pool (pool_block=True, so exceeding it silently serializes
+    # rather than erroring, so this is a real-concurrency cap, not a correctness one). Setting
     # this to 1 restores today's fully serial embed() and spawns no thread pool.
     semantic_embedding_concurrency: int = 4
 

config.yaml

@@ -12,8 +12,13 @@ version: 1
 # threads — the tree walk is GIL-serialized), which is why extraction now runs
 # in its own shared process pool instead — see extract_processes below. Raising
 # this knob buys disk-bound repo fan-out, not extraction throughput.
-# With semantic indexing enabled this is clamped to 2 — a MEMORY bound, since
-# embedding materialises a whole repo's chunks (~0.5-0.8 GB per worker).
+# With semantic indexing enabled this is clamped to 4 (issue #109 raised it from
+# 2, after re-deriving the memory model and confirming empirically against the
+# live dev job at N=4: peak self+children RSS landed at ~83% of the 0.7*container
+# memory budget, comfortably under and with more margin than N=2's own ~90%) — a
+# MEMORY bound, since embedding materialises a whole repo's chunks (structural
+# ~32 KB/chunk, resident ~40.1 KB/chunk measured; see effective_workers'
+# docstring and docs/perf/issue-109-measurements.md for the full derivation).
 # index_concurrency: 4
 
 # How many worker PROCESSES the job uses to extract symbols/edges (issue #108).
@@ -75,13 +80,23 @@ connections:
 # effective cap as `per-repo override OR global`.
 #
 # Mind the memory math before raising one: buffered vectors are ~32 KB/chunk
-# (dim=1024, Python float-list storage), so 8000 ≈ 260 MB resident for the
-# duration of that repo's write. With semantic on, at most 2 workers run
-# concurrently (indexer/repo_config.py's effective_workers clamp), so a large
-# override multiplies straight into the job container's peak memory — e.g. two
-# repos overridden to 20000 concurrently is ≈1 GB just in vectors, on top of
-# the base per-worker cost. A repo that legitimately needs far more than that
-# needs the temp-table staging path (follow-up), not a bigger override.
+# structural (dim=1024, Python float-list storage) but ~40.1 KB/chunk RESIDENT
+# (measured, issue #109 — includes pymalloc overhead/fragmentation; use this
+# figure for headroom arithmetic), so 8000 ≈ 313 MiB resident for the duration of
+# that repo's write. With semantic on, at most 4 workers run concurrently
+# (indexer/repo_config.py's effective_workers clamp, raised from 2 by #109), so
+# a large override multiplies straight into the job container's peak memory —
+# e.g. two repos overridden to 20000 concurrently is ≈1.6 GB just in vectors, on
+# top of the base per-worker cost. A repo that legitimately needs far more than
+# that needs the temp-table staging path (follow-up), not a bigger override.
+#
+# The derived per-worker chunk-cap ceiling, from a full container-memory model
+# (issue #109; docs/perf/issue-109-measurements.md §12 — pinned at N=2 there
+# only to break a circularity in solving for C from a formula whose dominant
+# term IS C, not a claim about the adopted concurrency): ≈73,300 chunks at
+# N=2, ≈36,700 at the shipped N=4 (both halve/double with N). The current
+# global default of 8000 uses well under a quarter of either budget, so it is
+# NOT the binding constraint and was left unchanged.
 # semantic_max_chunks_per_repo:
 #   "acme/huge-monorepo": 20000
 
@@ -98,20 +113,24 @@ connections:
 # (the default is 8000). It is NOT the per-repo `semantic_max_chunks_per_repo` MAP
 # above — that spot-overrides individual repos and still wins over this global. Use
 # this to raise the floor everyone inherits; use the map for the outliers. The same
-# ~32 KB/chunk memory math and 2-worker clamp above apply here, magnified: raising
-# the global lifts the buffer cost for EVERY concurrently-indexing repo at once.
+# ~32 KB structural / ~40.1 KB resident per-chunk memory math and 4-worker clamp
+# above apply here, magnified: raising the global lifts the buffer cost for EVERY
+# concurrently-indexing repo at once.
 #
 # `enabled: false` makes the job a true semantic no-op (no embedder built, no
-# chunking, the 2-worker memory clamp not applied) even if the env says enabled —
+# chunking, the 4-worker memory clamp not applied) even if the env says enabled —
 # the fastest way to turn semantic off for the job alone.
 #
 # `embedding_concurrency` (#107) is in-flight embedding requests PER WORKER, sent
 # via a ThreadPoolExecutor that preserves submission order — vectors always come
 # back in the order their texts were sent, regardless of which request finishes
 # first. Total in-flight gateway requests for the job is workers x concurrency:
-# 2 x 4 = 8 at this default, 2 x 8 = 16 at the max of 8, both under the SDK's
-# 20-connection pool. Set to 1 to restore fully serial embedding (no thread pool
-# spawned at all) if you need to roll back.
+# 4 x 4 = 16 at this default, 4 x 8 = 32 at the max of 8 — the latter now EXCEEDS
+# the SDK's 20-connection pool (issue #109 raised workers from 2 to 4; this
+# combination was not possible before). Lower embedding_concurrency if raising it
+# alongside a near-ceiling index_concurrency. Set embedding_concurrency to 1 to
+# restore fully serial embedding (no thread pool spawned at all) if you need to
+# roll back.
 # semantic:
 #   enabled: true
 #   max_chunks_per_repo: 8000