WIP

AGENTS.md

@@ -60,31 +60,42 @@ Client → large-model-proxy → [Service Process]
 
 ### Source Files
 
-| File                              | Purpose                                                                                     |
-| --------------------------------- | ------------------------------------------------------------------------------------------- |
-| `main.go`                         | Entry point, signal handling, and ResourceManager bookkeeping (connection counting, lookup) |
-| `config.go`                       | Configuration loading, validation, and defaults (JSONC parsing)                             |
-| `connection.go`                   | Per-service TCP listeners, client connection handling, and bidirectional traffic forwarding |
-| `service.go`                      | Service lifecycle: on-demand start, health checks, and connecting to a running service      |
-| `service_process.go`              | Service process spawning/stopping, output logging, and process-exit monitoring              |
-| `resources.go`                    | Resource reservation, LRU eviction, and resource release logic                              |
-| `openai_api.go`                   | Unified OpenAI API server and request routing to backends by model name                     |
-| `management_api.go`               | Management HTTP server and embedded web dashboard assets                                    |
-| `monitor_resources.go`            | Resource availability monitoring and change broadcasting                                    |
-| `monitor_process_hook.go`         | Test-only synchronization hook for process-exit timing (compiled with the `testhooks` tag)  |
-| `monitor_process_hook_default.go` | No-op production stub for `monitor_process_hook.go`                                         |
-| `tty.go`                          | TTY/terminal handling utilities                                                             |
+| File                              | Purpose                                                                                                   |
+| --------------------------------- | --------------------------------------------------------------------------------------------------------- |
+| `main.go`                         | Entry point, signal handling, and ResourceManager bookkeeping (connection counting, lookup)               |
+| `config.go`                       | Configuration loading, validation, and defaults (JSONC parsing)                                           |
+| `connection.go`                   | Per-service TCP listeners, client connection handling, and bidirectional traffic forwarding               |
+| `service.go`                      | Service lifecycle: on-demand start, health checks, and connecting to a running service                    |
+| `service_process.go`              | Service process spawning/stopping, output logging, and process-exit monitoring                            |
+| `resources.go`                    | Resource reservation, LRU eviction, and resource release logic                                            |
+| `openai_api.go`                   | Unified OpenAI API server and request routing to backends by model name                                   |
+| `port_allocation.go`              | Automatic listen port selection with SQLite persistence of last used ports                                |
+| `context_router.go`               | Context-based routing: shared-port service tiers, request size measurement, per-connection tier switching |
+| `http_request_splitter.go`        | Incremental HTTP request framing from a raw TCP stream (used by context routing)                          |
+| `tokenizer.go`                    | Named token counters (heuristic, per model family) used to measure request context sizes                  |
+| `management_api.go`               | Management HTTP server and embedded web dashboard assets                                                  |
+| `monitor_resources.go`            | Resource availability monitoring and change broadcasting                                                  |
+| `monitor_process_hook.go`         | Test-only synchronization hook for process-exit timing (compiled with the `testhooks` tag)                |
+| `monitor_process_hook_default.go` | No-op production stub for `monitor_process_hook.go`                                                       |
+| `tty.go`                          | TTY/terminal handling utilities                                                                           |
 
 ### Test Files
 
-| File                        | Purpose                                             |
-| --------------------------- | --------------------------------------------------- |
-| `main_test.go`              | Core proxy integration tests                        |
-| `config_test.go`            | Configuration parsing and validation tests          |
-| `management_api_test.go`    | Management API endpoint tests                       |
-| `monitor_resources_test.go` | Resource monitoring tests                           |
-| `util_test.go`              | Shared test utilities and helpers                   |
-| `test-server/main.go`       | Simulated backend service used in integration tests |
+| File                                 | Purpose                                                                              |
+| ------------------------------------ | ------------------------------------------------------------------------------------ |
+| `main_test.go`                       | Core proxy integration tests                                                         |
+| `config_test.go`                     | Configuration parsing and validation tests                                           |
+| `management_api_test.go`             | Management API endpoint tests                                                        |
+| `monitor_resources_test.go`          | Resource monitoring tests                                                            |
+| `port_allocation_test.go`            | Listen port store, allocator and config resolution tests                             |
+| `auto_listen_port_e2e_test.go`       | End-to-end automatic port selection test (reuse and reassignment)                    |
+| `context_router_test.go`             | Context routing tier selection and request unit counting tests                       |
+| `context_routing_connection_test.go` | In-process tests for the routed connection handler (routing, switching, passthrough) |
+| `context_routing_e2e_test.go`        | End-to-end context routing test through the real proxy binary                        |
+| `http_request_splitter_test.go`      | HTTP request framing tests                                                           |
+| `tokenizer_test.go`                  | Token counter tests                                                                  |
+| `util_test.go`                       | Shared test utilities and helpers                                                    |
+| `test-server/main.go`                | Simulated backend service used in integration tests                                  |
 
 ### Other Key Files
 

README.md

@@ -164,6 +164,7 @@ Below is a breakdown of what this configuration does:
 6. When ComfyUI is no longer in use, its container will be killed using the `docker kill comfyui` command. Other services will be terminated normally.
 7. `StartupTimeoutMilliseconds` in starting ComfyUI makes large-model-proxy wait up to 60 seconds before giving up and considering the ComfyUI startup failed (as opposed to the default value of 10 minutes).
 8. Service URLs are configured as follows:
+
    - **Automatic1111**: Uses the default URL template (`DefaultServiceUrl`) which resolves to `http://localhost:7860/`
    - **Gemma27B**: Uses a custom static URL `http://gemma-proxy-server/` (no port templating)
    - **Qwen2.5-7B-Instruct**: Explicitly set to `null`, so no URL will be generated even though a default is available
@@ -196,6 +197,132 @@ Currently, the following OpenAI API endpoints are supported:
 - `/v1/models/{model}`
 - More to come
 
+## Context-based routing
+
+Several services can share a single `ListenPort` to form a set of context-size
+tiers. The proxy then routes every request to the service with the **smallest
+context size that still fits** the request, and transparently switches to the
+next larger service once a request outgrows the current one — even in the
+middle of a keep-alive connection. This makes it possible to run, for
+example, a fast 4k-context instance of a model for short conversations and a
+bigger 32k-context instance that is only loaded when conversations actually
+grow that long:
+
+```jsonc
+{
+  "Services": [
+    {
+      "Name": "Qwen3-8B-4k",
+      "ListenPort": "8085",
+      "ProxyTargetHost": "localhost",
+      "ProxyTargetPort": "18085",
+      "Command": "llama-server",
+      "Args": "-m Qwen3-8B.gguf -c 4096 --port 18085",
+      "ContextSize": 4096, // context window in tokens
+      "Tokenizer": "qwen3.8",
+      "ResourceRequirements": { "VRAM-GPU-1": 9000 },
+    },
+    {
+      "Name": "Qwen3-8B-32k",
+      "ListenPort": "8085", // same port: joins the same routing group
+      "ProxyTargetHost": "localhost",
+      "ProxyTargetPort": "18086",
+      "Command": "llama-server",
+      "Args": "-m Qwen3-8B.gguf -c 32768 --port 18086",
+      "ContextSize": 32768,
+      "Tokenizer": "qwen3.8",
+      "ResourceRequirements": { "VRAM-GPU-1": 22000 },
+    },
+  ],
+}
+```
+
+Rules for a shared port:
+
+- Every service on the port must define a context size: either `ContextSize`
+  (tokens, together with a `Tokenizer`) or `ContextSizeBytes` (raw bytes).
+- All services in the group must use the same unit (tokens or bytes), the
+  same tokenizer, and unique context sizes. Starting a larger service follows
+  the usual resource logic: it may have to evict the smaller one (or any other
+  LRU service) first.
+
+### Token counting
+
+`Tokenizer` selects a registered token counter used to measure requests.
+Built-in counters: `qwen3.8` (alias `qwen3`) and `gemma4` (alias `gemma3`).
+Counting is a per-model-family approximation (character-class segmentation
+with per-family ratios) rather than an exact BPE implementation: it is close
+enough for tier selection but tends to slightly overestimate, so configure
+context sizes with headroom for the model's output tokens. Adding a counter
+for another model family is a single `RegisterTokenCounter` call in
+`tokenizer.go`.
+
+For services measured in `ContextSizeBytes`, the raw byte length of the
+request's text content is used instead of a token count — useful for models
+without a known tokenizer or for non-OpenAI-style HTTP backends.
+
+### How requests are measured
+
+Only the text content of the request body is counted: `messages[].content`
+(including content-part arrays, where non-text parts are skipped), `prompt`
+and `input` fields of OpenAI-style JSON requests. Bodies that are not
+recognizable JSON are measured as raw text. Requests that cannot be framed at
+all (no `Content-Length` and not chunked), non-HTTP traffic, and clients that
+send nothing for 60 seconds after connecting are blindly forwarded to the
+smallest tier.
+
+### Behavior details
+
+- Routing decisions happen per request. Switching to a larger tier closes the
+  current service connection, starts the larger service (with the usual
+  transparent startup delay for the client) and continues forwarding on the
+  same client connection. The smaller service then idles out on its own.
+- Switching only ever goes upward within a connection: a small follow-up
+  request (model list, new conversation on the same connection) stays on the
+  current tier so auxiliary requests cannot thrash services.
+- Requests larger than every configured tier are routed to the largest tier
+  (which will report the context-length error to the client).
+- A connection that outgrows all tiers keeps using the largest one.
+- Like all HTTP/1.1 intermediaries that re-route per request, this assumes
+  clients do not pipeline requests (every real HTTP client sends the next
+  request only after reading the previous response).
+
+## Automatic listen port selection
+
+A service can ask the proxy to pick its listen port automatically:
+
+```jsonc
+{
+  "Name": "Qwen3-8B",
+  "ListenPort": "auto", // any free port, remembered across restarts
+  "ProxyTargetHost": "localhost",
+  "ProxyTargetPort": "18085",
+  "Command": "llama-server",
+  "Args": "-m Qwen3-8B.gguf --port 18085",
+}
+```
+
+At startup the proxy first tries the port this service used last time, so
+clients keep working across restarts. If that port is no longer free (or the
+service has no history yet), a free port is requested from the OS; ports
+already used elsewhere in the same configuration are never picked. The chosen
+port is then persisted in a small SQLite database
+(`AutoListenPortDatabasePath`, default `auto-listen-ports.db` in the working
+directory, keyed by service name), and the port is bound immediately when
+selected so it cannot be lost to another process before the proxy starts
+listening on it.
+
+Ways to discover the assigned port:
+
+- the startup log line `[ServiceName] Automatically selected listen port N`
+  (and the usual `Listening on port N` line),
+- the `listen_port` field of the service in the Management API `/status`
+  response (also used for the `{{.PORT}}` variable in `ServiceUrl` templates),
+- the SQLite database itself.
+
+Each service with `ListenPort: "auto"` gets a port of its own. Services that
+share one port for context-based routing must use explicit numeric ports.
+
 ## Management API
 
 The management API is a simple HTTP API that allows you to get the status of the proxy and the services it is proxying.

auto_listen_port_e2e_test.go

@@ -0,0 +1,153 @@
+package main
+
+import (
+	"fmt"
+	"net"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// TestAutoListenPortSelection verifies automatic listen port selection end to
+// end: a service configured with ListenPort "auto" gets a free port, the
+// choice is persisted to the SQLite database, a restart reuses the same port,
+// and a restart while the old port is occupied picks (and persists) a new one.
+func TestAutoListenPortSelection(t *testing.T) {
+	t.Parallel()
+
+	repoDir, err := os.Getwd()
+	require.NoError(t, err)
+	workDir := t.TempDir()
+	databasePath := filepath.Join(workDir, defaultAutoListenPortDatabasePath)
+
+	testConfig := Config{
+		Services: []ServiceConfig{
+			{
+				Name:            "auto-service",
+				ListenPort:      autoListenPort,
+				ProxyTargetHost: "localhost",
+				ProxyTargetPort: "12160",
+				Command:         "./test-server/test-server",
+				Args:            "-p 12160",
+				Workdir:         repoDir,
+			},
+		},
+		ManagementApi: ManagementApi{ListenPort: "4160"},
+	}
+	StandardizeConfigNamesAndPaths(&testConfig, t.Name())
+	serviceName := testConfig.Services[0].Name
+	testConfig.Services[0].LogFilePath = filepath.Join(workDir, "auto-service.log")
+	configFilePath := createTempConfig(t, testConfig)
+
+	startProxyInstance := func() (*stopHandle, error) {
+		waitChannel := make(chan error, 1)
+		cmd, err := startLargeModelProxy("auto-listen-port", configFilePath, workDir, waitChannel)
+		if err != nil {
+			return nil, err
+		}
+		return &stopHandle{cmd: cmd, waitChannel: waitChannel}, nil
+	}
+
+	waitForAssignedPort := func() int {
+		t.Helper()
+		deadline := time.Now().Add(15 * time.Second)
+		for {
+			store, err := openListenPortStore(databasePath)
+			if err == nil {
+				port, found, getErr := store.getLastUsedPort(serviceName)
+				_ = store.close()
+				if getErr == nil && found {
+					return port
+				}
+			}
+			if time.Now().After(deadline) {
+				t.Fatalf("service %s never got a port persisted in %s", serviceName, databasePath)
+			}
+			time.Sleep(50 * time.Millisecond)
+		}
+	}
+
+	verifyServiceReachableThroughPort := func(port int) {
+		t.Helper()
+		connection, err := net.DialTimeout("tcp", fmt.Sprintf("localhost:%d", port), 10*time.Second)
+		if err != nil {
+			t.Fatalf("failed to connect to automatically selected port %d: %v", port, err)
+		}
+		defer func() { _ = connection.Close() }()
+		_ = connection.SetReadDeadline(time.Now().Add(15 * time.Second))
+		pid := readPidFromOpenConnection(t, connection)
+		assert.True(t, isProcessRunning(pid))
+	}
+
+	// First run: a port is selected and persisted
+	proxy, err := startProxyInstance()
+	require.NoError(t, err)
+	firstPort := waitForAssignedPort()
+	verifyServiceReachableThroughPort(firstPort)
+	status := getStatusFromManagementAPI(t, "localhost:4160")
+	serviceStatus := findServiceInStatusResponse(status, serviceName)
+	require.NotNil(t, serviceStatus)
+	assert.Equal(t, fmt.Sprintf("%d", firstPort), serviceStatus.ListenPort,
+		"the management API must report the resolved port")
+	require.NoError(t, proxy.stop())
+	waitForPortClosed(t, "localhost:4160", 15*time.Second)
+
+	// Second run with the same database: the same port must be reused
+	proxy, err = startProxyInstance()
+	require.NoError(t, err)
+	secondPort := waitForAssignedPort()
+	assert.Equal(t, firstPort, secondPort, "a free persisted port must be reused on restart")
+	verifyServiceReachableThroughPort(secondPort)
+	require.NoError(t, proxy.stop())
+	waitForPortClosed(t, "localhost:4160", 15*time.Second)
+
+	// Third run with the persisted port occupied by someone else: a new port
+	// must be selected and persisted instead
+	blocker, err := net.Listen("tcp", fmt.Sprintf(":%d", firstPort))
+	require.NoError(t, err)
+	proxy, err = startProxyInstance()
+	require.NoError(t, err)
+	thirdPort := waitForAssignedPort()
+	assert.NotEqual(t, firstPort, thirdPort, "an occupied persisted port must not be reused")
+	verifyServiceReachableThroughPort(thirdPort)
+	require.NoError(t, proxy.stop())
+	_ = blocker.Close()
+	waitForPortClosed(t, "localhost:4160", 15*time.Second)
+
+	// The database must reflect the latest choice
+	store, err := openListenPortStore(databasePath)
+	require.NoError(t, err)
+	port, found, getErr := store.getLastUsedPort(serviceName)
+	require.NoError(t, getErr)
+	require.NoError(t, store.close())
+	assert.True(t, found)
+	assert.Equal(t, thirdPort, port)
+}
+
+type stopHandle struct {
+	cmd         *exec.Cmd
+	waitChannel chan error
+}
+
+func (h *stopHandle) stop() error {
+	return stopApplication(h.cmd, h.waitChannel)
+}
+
+func waitForPortClosed(t *testing.T, address string, timeout time.Duration) {
+	t.Helper()
+	deadline := time.Now().Add(timeout)
+	for {
+		if err := checkPortClosed(address); err == nil {
+			return
+		}
+		if time.Now().After(deadline) {
+			t.Fatalf("port %s did not close within %s", address, timeout)
+		}
+		time.Sleep(50 * time.Millisecond)
+	}
+}