diff --git a/cmd/svcinit/main.go b/cmd/svcinit/main.go index 6114f6c..09d7878 100644 --- a/cmd/svcinit/main.go +++ b/cmd/svcinit/main.go @@ -9,7 +9,6 @@ import ( "fmt" "io" "log" - "maps" "math" "net" "os" @@ -125,10 +124,20 @@ func main() { listener, err := net.Listen("tcp", "127.0.0.1:0") must(err) - ports, reservedPorts, err := assignPorts(unversionedSpecs) + portsMap, servicesMap, reservedPorts, err := assignPorts(unversionedSpecs) must(err) defer closeReservedPorts(reservedPorts) + // Expose the rich port/service maps. These are inherited by both the test binary and + // all child services since they spawn with os.Environ() as their base. + serializedPortsMap, err := portsMap.Marshal() + must(err) + os.Setenv("ITEST_PORTS_MAP", string(serializedPortsMap)) + + serializedServicesMap, err := servicesMap.Marshal() + must(err) + os.Setenv("ITEST_SERVICES_MAP", string(serializedServicesMap)) + svcctlPort := listener.Addr().(*net.TCPAddr).Port svcctlPortStr := strconv.Itoa(svcctlPort) os.Setenv("SVCCTL_PORT", svcctlPortStr) @@ -139,7 +148,7 @@ func main() { defer os.Remove("/tmp/svcctl_port") } - serviceSpecs, err := augmentServiceSpecs(unversionedSpecs, ports, svcctlPortStr) + serviceSpecs, err := augmentServiceSpecs(unversionedSpecs, portsMap, svcctlPortStr) must(err) ctx, cancelFunc := context.WithCancel(context.Background()) @@ -159,7 +168,7 @@ func main() { go func() { defer listener.Close() - err := svcctl.Serve(ctx, listener, r, ports, servicesErrCh) + err := svcctl.Serve(ctx, listener, r, portsMap, servicesMap, servicesErrCh) if err != nil { log.Fatalf("svcctl.Serve: %v", err) } @@ -212,7 +221,7 @@ func main() { // Bazel's args attribute converts $$ to $, so args arrive with // single-$ placeholders (e.g. ${@@//:svc}) unlike env/spec files // which preserve the literal $$ since they're read from JSON. - argReplacements := buildReplacements(ports, "${") + argReplacements := buildReplacements(portsMap, "${") testArgs := make([]string, len(os.Args[1:])) for i, arg := range os.Args[1:] { testArgs[i] = replaceAll(arg, argReplacements) @@ -220,7 +229,7 @@ func main() { testPath, err := runfiles.Rlocation(os.Getenv("SVCINIT_TEST_RLOCATION_PATH")) must(err) - testEnv, err := buildTestEnv(ports) + testEnv, err := buildTestEnv(portsMap) must(err) fmt.Println("") @@ -287,7 +296,7 @@ func main() { unversionedSpecs, err := readServiceSpecs(serviceSpecsPath) must(err) - serviceSpecs, err := augmentServiceSpecs(unversionedSpecs, ports, svcctlPortStr) + serviceSpecs, err := augmentServiceSpecs(unversionedSpecs, portsMap, svcctlPortStr) must(err) testCancel() @@ -376,32 +385,77 @@ func readServiceSpecs( func assignPorts( serviceSpecs map[string]svclib.ServiceSpec, ) ( - svclib.Ports, map[string][]io.Closer, error, + svclib.PortsMap, svclib.ServicesMap, map[string][]io.Closer, error, ) { var toClose []io.Closer reservedPorts := map[string][]io.Closer{} - ports := svclib.Ports{} + portsMap := svclib.PortsMap{} + servicesMap := svclib.ServicesMap{} + + // Tracks which service bound each port target, so we can enforce that a port is only + // ever bound once. + boundBy := map[string]string{} + + // register binds a resolved port under its target label and every alias, in the rich + // port/service maps. The legacy string->port view (ASSIGNED_PORTS, substitution, + // /v0/port) is derived from portsMap on demand. + register := func(serviceLabel, portName, hostname, portStr, target string, aliases []string) { + info := svclib.BindingInfo{ + Origin: net.JoinHostPort(hostname, portStr), + Hostname: hostname, + Port: portStr, + } + + keys := append([]string{target}, aliases...) + for _, key := range keys { + portsMap[key] = info + } + servicesMap.Set(serviceLabel, portName, info) + } for label, spec := range serviceSpecs { - namedPorts := maps.Clone(spec.NamedPorts) - if spec.AutoassignPort { - namedPorts[""] = spec.Port + if len(spec.PortBindings) == 0 { + continue } - // Note, this can cause collisions. So be careful! - // To avoid port collisions, set so_reuseport_aware on the service definition - // and use SO_REUSEPORT on Unix or SO_REUSEADDR on Windows in your services. - for portName, port := range namedPorts { + hostname := spec.Hostname + if hostname == "" { + hostname = "127.0.0.1" + } + + for _, binding := range spec.PortBindings { + if other, ok := boundBy[binding.Target]; ok && other != label { + return nil, nil, nil, fmt.Errorf( + "port %q is bound by multiple services: %q and %q. A port may only be bound once", + binding.Target, other, label, + ) + } + boundBy[binding.Target] = label + + // External services are not managed by us; their ports are reachable as-is at the FQDN. + if spec.Type == "external_service" { + if !terseOutput { + log.Printf("Registering external port %s for %s (%s)\n", binding.Value, binding.Target, hostname) + } + register(label, binding.Name, hostname, binding.Value, binding.Target, binding.Aliases) + continue + } + + // Internal service: reserve the port so we can discover an autoassigned one and hold it. + // Note, this can cause collisions. So be careful! + // To avoid port collisions, set so_reuseport_aware on the service definition + // and use SO_REUSEPORT on Unix or SO_REUSEADDR on Windows in your services. var reservedPort io.Closer - var err error + var portStr string if spec.SoReuseportAware { - requestedPort, parseErr := strconv.Atoi(port) + requestedPort, parseErr := strconv.Atoi(binding.Value) if parseErr != nil || requestedPort < 0 || requestedPort > 65535 { - return nil, nil, fmt.Errorf("invalid port %q for %s", port, label) + return nil, nil, nil, fmt.Errorf("invalid port %q for %s", binding.Value, label) } - reservedPort, port, err = reserveReusablePort(requestedPort) + var err error + reservedPort, portStr, err = reserveReusablePort(requestedPort) if err != nil { - return nil, nil, err + return nil, nil, nil, err } } else { // We do a bit of a dance here to set SO_LINGER to 0. For details, see @@ -422,42 +476,23 @@ func assignPorts( }, } - listener, listenErr := lc.Listen(context.Background(), "tcp", "127.0.0.1:"+port) - if listenErr != nil { - return nil, nil, listenErr + listener, err := lc.Listen(context.Background(), "tcp", "127.0.0.1:"+binding.Value) + if err != nil { + return nil, nil, nil, err } - _, port, err = net.SplitHostPort(listener.Addr().String()) + _, portStr, err = net.SplitHostPort(listener.Addr().String()) if err != nil { listener.Close() - return nil, nil, err + return nil, nil, nil, err } reservedPort = listener } - qualifiedPortName := label - if portName != "" { - qualifiedPortName += "." + portName - } - if !terseOutput { - log.Printf("Assigning port %s to %s\n", port, qualifiedPortName) + log.Printf("Assigning port %s to %s\n", portStr, binding.Target) } - ports.Set(qualifiedPortName, port) - - { - // TODO(zbarsky): Clean this up after April 2026 - qualifiedPortName := label - if portName != "" { - qualifiedPortName += ":" + portName - } - - if !terseOutput { - log.Printf("Assigning port %s to %s\n", port, qualifiedPortName) - } - - ports.Set(qualifiedPortName, port) - } + register(label, binding.Name, hostname, portStr, binding.Target, binding.Aliases) if !spec.SoReuseportAware { toClose = append(toClose, reservedPort) @@ -468,30 +503,24 @@ func assignPorts( } for _, reservedPort := range toClose { - err := reservedPort.Close() - if err != nil { - return nil, nil, err + if err := reservedPort.Close(); err != nil { + return nil, nil, nil, err } } + // Resolve service-group port aliases (re-exports of another service's port). for label, spec := range serviceSpecs { for portName, aliasedTo := range spec.PortAliases { - qualifiedPortName := label + // Zero value if the aliased target has no rich info; Port will be "". + info := portsMap[aliasedTo] + + qualifiedDot := label if portName != "" { - qualifiedPortName += "." + portName + qualifiedDot += "." + portName } + portsMap[qualifiedDot] = info - ports.Set(qualifiedPortName, ports[aliasedTo]) - - { - // TODO(zbarsky): Clean this up after April 2026 - qualifiedPortName := label - if portName != "" { - qualifiedPortName += ":" + portName - } - - ports.Set(qualifiedPortName, ports[aliasedTo]) - } + servicesMap.Set(label, portName, info) } } @@ -499,12 +528,12 @@ func assignPorts( // Give the kernel a bit of time to figure out what we've done. time.Sleep(10 * time.Millisecond) - serializedPorts, err := ports.Marshal() + serializedPorts, err := portsMap.AssignedPorts().Marshal() if err != nil { - return nil, nil, err + return nil, nil, nil, err } os.Setenv("ASSIGNED_PORTS", string(serializedPorts)) - return ports, reservedPorts, nil + return portsMap, servicesMap, reservedPorts, nil } func closeReservedPorts(reservedPorts map[string][]io.Closer) { @@ -519,14 +548,11 @@ func closeReservedPorts(reservedPorts map[string][]io.Closer) { func augmentServiceSpecs( serviceSpecs map[string]svclib.ServiceSpec, - ports svclib.Ports, + portsMap svclib.PortsMap, svcctlPort string, ) ( map[string]svclib.VersionedServiceSpec, error, ) { - tmpDir := os.Getenv("TMPDIR") - socketDir := os.Getenv("SOCKET_DIR") - versionedServiceSpecs := make(map[string]svclib.VersionedServiceSpec, len(serviceSpecs)) for label, serviceSpec := range serviceSpecs { s := svclib.VersionedServiceSpec{ @@ -538,11 +564,21 @@ func augmentServiceSpecs( continue } - exePath, err := runfiles.Rlocation(s.Exe) - if err != nil { - return nil, err + // Env is always present for spawned/external specs, but normalize defensively so the + // substitution and SVCCTL_PORT write below can assume a non-nil map. + if s.Env == nil { + s.Env = map[string]string{} + } + + // External services are not spawned, but their health-check address/args may still + // reference ports/origins, so they go through substitution below. + if s.Type != "external_service" { + exePath, err := runfiles.Rlocation(s.Exe) + if err != nil { + return nil, err + } + s.Exe = exePath } - s.Exe = exePath if s.HealthCheck != "" { healthCheckPath, err := runfiles.Rlocation(serviceSpec.HealthCheck) @@ -568,7 +604,7 @@ func augmentServiceSpecs( s.Color = logger.Colorize(s.Label) if s.AutoassignPort { - port := ports[s.Label] + port := portsMap.Port(s.Label) for i := range s.ServiceSpec.Args { s.Args[i] = strings.ReplaceAll(s.Args[i], "$${PORT}", port) } @@ -585,18 +621,7 @@ func augmentServiceSpecs( versionedServiceSpecs[label] = s } - replacements := make([]Replacement, 0, 2+len(ports)) - replacements = append(replacements, - Replacement{Old: "$${TMPDIR}", New: tmpDir}, - Replacement{Old: "$${SOCKET_DIR}", New: socketDir}, - ) - - for label, port := range ports { - replacements = append(replacements, Replacement{ - Old: "$${" + label + "}", - New: port, - }) - } + replacements := buildReplacements(portsMap, "$${") replaceAllPorts := func(s string) string { for _, r := range replacements { @@ -630,17 +655,19 @@ type Replacement struct { // buildReplacements creates port/env substitution pairs. // prefix is "$${" for values from JSON files (which preserve literal $$), // or "${" for values from Bazel args (where $$ is already collapsed to $). -func buildReplacements(ports svclib.Ports, prefix string) []Replacement { - replacements := make([]Replacement, 0, 2+len(ports)) +func buildReplacements(portsMap svclib.PortsMap, prefix string) []Replacement { + replacements := make([]Replacement, 0, 2+3*len(portsMap)) replacements = append(replacements, Replacement{Old: prefix + "TMPDIR}", New: os.Getenv("TMPDIR")}, Replacement{Old: prefix + "SOCKET_DIR}", New: os.Getenv("SOCKET_DIR")}, ) - for label, port := range ports { - replacements = append(replacements, Replacement{ - Old: prefix + label + "}", - New: port, - }) + for label, info := range portsMap { + replacements = append(replacements, + Replacement{Old: prefix + label + "}", New: info.Port}, + // Rich origin/hostname tokens. A "::" delimiter is used since it can't appear in a label. + Replacement{Old: prefix + label + "::origin}", New: info.Origin}, + Replacement{Old: prefix + label + "::hostname}", New: info.Hostname}, + ) } return replacements } @@ -652,7 +679,7 @@ func replaceAll(s string, replacements []Replacement) string { return s } -func buildTestEnv(ports svclib.Ports) ([]string, error) { +func buildTestEnv(portsMap svclib.PortsMap) ([]string, error) { testEnvPath, err := runfiles.Rlocation(os.Getenv("SVCINIT_TEST_ENV_RLOCATION_PATH")) if err != nil { panic(err) @@ -669,7 +696,7 @@ func buildTestEnv(ports svclib.Ports) ([]string, error) { panic(err) } - replacements := buildReplacements(ports, "$${") + replacements := buildReplacements(portsMap, "$${") // Note, this can technically specify the same var multiple times. // Last one wins - hope that's what you wanted! diff --git a/docs/itest.md b/docs/itest.md index 28f1b59..6d9c322 100644 --- a/docs/itest.md +++ b/docs/itest.md @@ -29,7 +29,7 @@ or `SO_REUSEADDR` on Windows. The option must be set before binding the service # Service control The service manager exposes a HTTP server on `http://127.0.0.1:{SVCCTL_PORT}`. It can be used to -start / stop services during a test run. There are currently 5 API endpoints available. +start / stop services during a test run. There are currently 7 API endpoints available. All of them are GET requests: 1. `/v0/healthcheck?service={label}`: Returns 200 if the service is healthy, 503 otherwise. @@ -38,10 +38,36 @@ All of them are GET requests: You can optionally specify the signal to send to the service (valid values: SIGTERM and SIGKILL). 4. `/v0/wait?service={label}`: Wait for the service to exit and returns the exit code in the body. 5. `/v0/port?service={label}`: Returns the assigned port for the given label. May be a named port. +6. `/v0/ports`: Returns the full `ITEST_PORTS_MAP` as JSON (every port target/alias -> binding info). +7. `/v0/services`: Returns the full `ITEST_SERVICES_MAP` as JSON (every service -> port name -> binding info). In `bazel run` mode, the service manager will write the value of `SVCCTL_PORT` to `/tmp/svcctl_port`. This can be used in conjunction with the `/v0/port` API to let other tools interact with the managed services. +# Ports, hostnames, and external services + +Ports can be declared as first-class targets with `itest_port`. A port is a handle whose value is an `int` +build-setting flag (`0` = autoassign); the hostname is supplied by the internal or external service that +binds it. The value can be pinned from the command line via `--//pkg:my_port=8080`. A given port may only +ever be bound once. + +`itest_external_service` points at a production or production-like instance reachable at a fixed FQDN. +It provides the same information as an `itest_service`, so it can be swapped in for a local service using +Bazel `select()` to run a test suite against production-like instances. External services are never started +or stopped by the service manager; if a health check is configured, it is run to verify reachability. + +Two environment variables describe all bound ports/services (both internal and external). They are injected +into the test binary and every child service, and are also available through the `/v0/ports` and `/v0/services` +control APIs: + +- `ITEST_PORTS_MAP`: a JSON object keyed by port target label (and aliases): + `{"@@//pkg:my_port": {"origin": "127.0.0.1:54321", "hostname": "127.0.0.1", "port": "54321"}, ...}` +- `ITEST_SERVICES_MAP`: a JSON object keyed by service target label, then by port name (and aliases): + `{"@@//pkg:my_service": {"http": {"origin": "...", "hostname": "...", "port": "..."}}, ...}` + +The legacy `ASSIGNED_PORTS` env var, the `GET_ASSIGNED_PORT_BIN` helper, and the `/v0/port` endpoint all +continue to work as before. + ## itest_service @@ -49,10 +75,10 @@ This can be used in conjunction with the `/v0/port` API to let other tools inter 
 load("@rules_itest//:itest.bzl", "itest_service")
 
-itest_service(name, autoassign_port, data, deps, enforce_graceful_shutdown, env, exe,
+itest_service(name, autoassign_port, data, deps, hostname, enforce_graceful_shutdown, env, exe,
               expected_start_duration, health_check, health_check_args, health_check_interval,
               health_check_timeout, hot_reloadable, http_health_check_address, named_ports,
-              port, shutdown_signal, shutdown_timeout, so_reuseport_aware)
+              port, ports, shutdown_signal, shutdown_timeout, so_reuseport_aware)
An itest_service is a binary that is intended to run for the duration of the integration test. Examples include databases, HTTP/RPC servers, queue consumers, external service mocks, etc. @@ -67,6 +93,7 @@ All [common binary attributes](https://bazel.build/reference/be/common-definitio | name | A unique name for this target. | Name | required | | | deps | Services/tasks that must be started before this service/task can be started. Can be `itest_service`, `itest_task`, or `itest_service_group`. | List of labels | optional | `[]` | | data | - | List of labels | optional | `[]` | +| hostname | The host that this service's ports are reachable on. Defaults to `127.0.0.1` for locally-managed services. | String | optional | `"127.0.0.1"` | | autoassign_port | If true, the service manager will pick a free port and assign it to the service. The port will be interpolated into `$${PORT}` in the service's `http_health_check_address` and `args`. It will also be exported under the target's fully qualified label in the service-port mapping.

The assigned ports for all services are available for substitution in `http_health_check_address` and `args` (in case one service needs the address for another one.) For example, the following substitution: `args = ["-client-addr", "127.0.0.1:$${@@//label/for:service}"]`

The service-port mapping is a JSON string -> string map propagated through the `ASSIGNED_PORTS` env var. For example, a port (as a string) can be retrieved with the following JS code: `JSON.parse(process.env["ASSIGNED_PORTS"])["@@//label/for:service"]`.

Alternately, the env will also contain the location of a binary that can return the port, for contexts without a readily-accessible JSON parser. For example, the following Bash command: `PORT=$($GET_ASSIGNED_PORT_BIN @@//label/for:service)` | Boolean | optional | `False` | | enforce_graceful_shutdown | If set to True, the service manager will fail the service_test if the service had to be forcefully killed if the signal was not SIGKILL and after the shutdown timeout elapsed.

This needs to be False to have coverage of your services but don't want a them to be graceful at shutdown | Label | optional | `"@rules_itest//:enforce_graceful_shutdown"` | | env | The service manager will merge these variables into the environment when spawning the underlying binary. | Dictionary: String -> String | optional | `{}` | @@ -78,13 +105,84 @@ All [common binary attributes](https://bazel.build/reference/be/common-definitio | health_check_timeout | The timeout to wait for the health check. The syntax is based on common time duration with a number, followed by the time unit. For example, `200ms`, `1s`, `2m`, `3h`, `4d`. If empty or not set, the health check will not have a timeout. | String | optional | `""` | | hot_reloadable | If set to True, the service manager will propagate ibazel's reload notification over stdin instead of restarting the service. See the ruleset docstring for more info on using ibazel | Boolean | optional | `False` | | http_health_check_address | If set, the service manager will send an HTTP request to this address to check if the service came up in a healthy state. This check will be retried until it returns a 200 HTTP code. When used in conjunction with autoassigned ports, `$${@@//label/for:service:port_name}` can be used in the address. Example: `http_health_check_address = "http://127.0.0.1:$${@@//label/for:service:port_name}",` | String | optional | `""` | -| named_ports | For each element of the list, the service manager will pick a free port and assign it to the service. The port's fully-qualified name is the service's fully-qualified label and the port name, separated by a colon. For example, a port assigned with `named_ports = ["http_port"]` will be assigned a fully-qualified name of `@@//label/for:service:http_port`.

Named ports are accessible through the service-port mapping. For more details, see `autoassign_port`. | List of strings | optional | `[]` | +| named_ports | For each element of the list, the service manager will pick a free port and assign it to the service. The port's fully-qualified name is the service's fully-qualified label and the port name, separated by a dot. For example, a port assigned with `named_ports = ["http_port"]` will be assigned a fully-qualified name of `@@//label/for:service.http_port`.

Named ports are accessible through the service-port mapping. For more details, see `autoassign_port`. | List of strings | optional | `[]` | | port | Internal. | Label | optional | `None` | +| ports | Maps `itest_port` targets that this service binds to a port name. The port name is used as the inner key in `ITEST_SERVICES_MAP`. The desired value for each port is carried by the `itest_port` target itself (an `int` flag, default `0` = autoassign), and can be pinned from the command line via `--//pkg:my_port=8080`. | Dictionary: Label -> String | optional | `{}` | | shutdown_signal | The signal to send to the service when it needs to be shut down. Valid values are: SIGTERM and SIGKILL. SIGTERM is necessary to have proper coverage of services which needs to be gracefully terminated | String | optional | `"SIGTERM"` | | shutdown_timeout | The duration to wait by default after sending the shutdown signal before forcefully killing the service. The syntax is based on common time duration with a number, followed by the time unit. For example, `200ms`, `1s`, `2m`, `3h`, `4d`. If not defined, the value of `_default_shutdown_timeout` will be used. | String | optional | `""` | | so_reuseport_aware | If set, the service manager keeps a bind-only reservation for the autoassigned port for the service manager's lifetime. The service binary must use SO_REUSEPORT on Unix or SO_REUSEADDR on Windows when binding it. This reduces the possibility of port collisions when running many service_tests in parallel, or when code binds port 0 without being aware of the port assignment mechanism.

Must only be set when `autoassign_port` is enabled or `named_ports` are used. | Boolean | optional | `False` | + + +## itest_external_service + +
+load("@rules_itest//:itest.bzl", "itest_external_service")
+
+itest_external_service(name, data, deferred, deps, hostname, expected_start_duration, health_check,
+                       health_check_args, health_check_interval, health_check_timeout, http_health_check_address,
+                       port_numbers, ports)
+
+ +An itest_external_service points at a production or production-like instance of a service that is +reachable at a fixed FQDN, rather than a binary that the service manager spawns locally. + +The service manager never starts or stops external services. If a health check is configured, it will be run +to verify the external service is reachable before dependent services/tests run. + +Because it provides the same information as an `itest_service`, it can be swapped in for a local service using +Bazel `select()` to run a test suite against production-like instances. + +**ATTRIBUTES** + + +| Name | Description | Type | Mandatory | Default | +| :------------- | :------------- | :------------- | :------------- | :------------- | +| name | A unique name for this target. | Name | required | | +| hostname | The fully-qualified domain name (FQDN) that this external service is reachable on, e.g. `my_service.test.mycompany.com`. | String | required | | +| ports | Maps `itest_port` targets that this external service exposes to a port name. Provide the literal port number for each name via `port_numbers`. | Dictionary: Label -> String | optional | `{}` | +| port_numbers | Maps each port name (from `ports`) to the literal port number it is reachable on at the FQDN. | Dictionary: String -> String | optional | `{}` | +| data | - | List of labels | optional | `[]` | +| deps | Services/tasks that must be available before this external service can be used. | List of labels | optional | `[]` | +| deferred | If set, the external service will not be health-checked on boot up. | Boolean | optional | `False` | +| http_health_check_address | If set, the service manager will send an HTTP request to this address to verify the external service is reachable. Port substitutions (including `$${::origin}`) are supported. | String | optional | `""` | +| health_check | If set, the service manager will execute this binary to verify the external service is reachable. | Label | optional | `None` | +| health_check_args | Arguments to pass to the health_check binary. Port substitutions are applied prior to execution. | List of strings | optional | `[]` | +| health_check_interval | The duration between each health check. | String | optional | `"200ms"` | +| health_check_timeout | The timeout to wait for the health check. If empty, the health check will not have a timeout. | String | optional | `""` | +| expected_start_duration | How long the service is expected to take before passing a health check. Failing checks before this elapses are not logged. | String | optional | `"0s"` | + + + + +## itest_port + +
+load("@rules_itest//:itest.bzl", "itest_port")
+
+itest_port(name, aliases, build_setting_default)
+
+ +Declares a port as a first-class target. A port is a handle: the host is supplied by the internal or +external service that binds it, while the desired value is carried by the port target itself as an `int` +build-setting flag (`0` = autoassign for internal services). A given port may only be bound by a single +service. + +Because the port is a flag, its value can be pinned from the command line, e.g. `--//pkg:my_port=8080`. + +Ports are always referenced by their target label (e.g. via `port` / `port_ref`), and are bound exactly once. + +**ATTRIBUTES** + + +| Name | Description | Type | Mandatory | Default | +| :------------- | :------------- | :------------- | :------------- | :------------- | +| name | A unique name for this target. | Name | required | | +| aliases | Additional keys that must resolve to this port in `ITEST_PORTS_MAP` and the `ASSIGNED_PORTS` map. Useful for keeping backwards-compatible references working. The port target's own label is always bound, in addition to any aliases. | List of strings | optional | `[]` | +| build_setting_default | The default value of this port flag. `0` means autoassign for the internal service that binds it. | Integer | optional | `0` | + + ## itest_service_group diff --git a/itest.bzl b/itest.bzl index b24ab3d..7d6bad4 100644 --- a/itest.bzl +++ b/itest.bzl @@ -3,6 +3,8 @@ load("@bazel_skylib//rules:common_settings.bzl", "int_flag") load( "//private:itest.bzl", + _itest_external_service = "itest_external_service", + _itest_port = "itest_port", _itest_service = "itest_service", _itest_service_group = "itest_service_group", _itest_task = "itest_task", @@ -43,6 +45,33 @@ def named_port_alias(label, name): """ return _to_relative_named_port(label, name) +def port_ref(label): + """References the assigned port of an `itest_port` target in the `args` or `env` of an `itest_service`/`itest_task`. + + This is equivalent to `port`, but is named to make it clear it points at an `itest_port` target. + """ + return "$${%s}" % _to_relative_port(label) + +def port_origin(label): + """References the origin (`:`) of an `itest_port` target in `args`/`env`/`http_health_check_address`.""" + return "$${%s::origin}" % _to_relative_port(label) + +def port_hostname(label): + """References the hostname (host) of an `itest_port` target in `args`/`env`/`http_health_check_address`.""" + return "$${%s::hostname}" % _to_relative_port(label) + +def itest_port(name, build_setting_default = 0, **kwargs): + """Declares a first-class port target. + + The port is an `int` build-setting flag; `0` (the default) means autoassign for the internal service + that binds it. The value can be pinned from the command line via `--//pkg:name=8080`. + """ + _itest_port( + name = name, + build_setting_default = build_setting_default, + **kwargs + ) + def itest_service(name, tags = [], hygienic = True, named_ports = [], **kwargs): if "port" in kwargs: fail("Do not specify `port`, instead set it via the `%s` flag" % (name + ".port")) @@ -76,6 +105,19 @@ def itest_service(name, tags = [], hygienic = True, named_ports = [], **kwargs): tags = tags, ) +def itest_external_service(name, tags = [], hygienic = True, **kwargs): + _itest_external_service( + name = name, + tags = tags + ["ibazel_notify_changes"], + **kwargs + ) + + if hygienic: + _hygiene_test( + name = name, + tags = tags, + ) + def itest_service_group(name, tags = [], hygienic = True, **kwargs): _itest_service_group( name = name, @@ -111,3 +153,10 @@ def _hygiene_test(name, **kwargs): ) service_test = _service_test + +# The underlying rules are exported so that they can be more easily extended by users +# (for example, to wrap them in their own macros). Prefer the macros above for normal usage. +itest_service_rule = _itest_service +itest_task_rule = _itest_task +itest_service_group_rule = _itest_service_group +itest_external_service_rule = _itest_external_service diff --git a/private/itest.bzl b/private/itest.bzl index ea51ac7..1a864c8 100644 --- a/private/itest.bzl +++ b/private/itest.bzl @@ -28,7 +28,7 @@ or `SO_REUSEADDR` on Windows. The option must be set before binding the service # Service control The service manager exposes a HTTP server on `http://127.0.0.1:{SVCCTL_PORT}`. It can be used to -start / stop services during a test run. There are currently 5 API endpoints available. +start / stop services during a test run. There are currently 7 API endpoints available. All of them are GET requests: 1. `/v0/healthcheck?service={label}`: Returns 200 if the service is healthy, 503 otherwise. @@ -37,9 +37,35 @@ All of them are GET requests: You can optionally specify the signal to send to the service (valid values: SIGTERM and SIGKILL). 4. `/v0/wait?service={label}`: Wait for the service to exit and returns the exit code in the body. 5. `/v0/port?service={label}`: Returns the assigned port for the given label. May be a named port. +6. `/v0/ports`: Returns the full `ITEST_PORTS_MAP` as JSON (every port target/alias -> binding info). +7. `/v0/services`: Returns the full `ITEST_SERVICES_MAP` as JSON (every service -> port name -> binding info). In `bazel run` mode, the service manager will write the value of `SVCCTL_PORT` to `/tmp/svcctl_port`. This can be used in conjunction with the `/v0/port` API to let other tools interact with the managed services. + +# Ports, hostnames, and external services + +Ports can be declared as first-class targets with `itest_port`. A port is a handle whose value is an `int` +build-setting flag (`0` = autoassign); the hostname is supplied by the internal or external service that +binds it. The value can be pinned from the command line via `--//pkg:my_port=8080`. A given port may only +ever be bound once. + +`itest_external_service` points at a production or production-like instance reachable at a fixed FQDN. +It provides the same information as an `itest_service`, so it can be swapped in for a local service using +Bazel `select()` to run a test suite against production-like instances. External services are never started +or stopped by the service manager; if a health check is configured, it is run to verify reachability. + +Two environment variables describe all bound ports/services (both internal and external). They are injected +into the test binary and every child service, and are also available through the `/v0/ports` and `/v0/services` +control APIs: + +- `ITEST_PORTS_MAP`: a JSON object keyed by port target label (and aliases): + `{"@@//pkg:my_port": {"origin": "127.0.0.1:54321", "hostname": "127.0.0.1", "port": "54321"}, ...}` +- `ITEST_SERVICES_MAP`: a JSON object keyed by service target label, then by port name (and aliases): + `{"@@//pkg:my_service": {"http": {"origin": "...", "hostname": "...", "port": "..."}}, ...}` + +The legacy `ASSIGNED_PORTS` env var, the `GET_ASSIGNED_PORT_BIN` helper, and the `/v0/port` endpoint all +continue to work as before. """ load("@bazel_lib//lib:paths.bzl", "to_rlocation_path") @@ -53,12 +79,89 @@ _ServiceGroupInfo = provider( }, ) +_PortInfo = provider( + doc = "A handle for a port that can be bound by an internal or external service.", + fields = { + "label": "The canonical (fully-qualified) label of the port target.", + "aliases": "Additional keys that must resolve to the same port for backwards compatibility.", + }, +) + def _collect_services(deps): services = {} for dep in deps: services |= dep[_ServiceGroupInfo].services return services +def _validate_unique_port_bindings(services): + """Ensures every port target is bound by at most one service.""" + seen = {} + for label, service in services.items(): + bindings = getattr(service, "port_bindings", None) + if not bindings: + continue + for binding in bindings: + if binding.target in seen and seen[binding.target] != label: + fail("Port %s is bound by multiple services: %s and %s. A port may only be bound once." % ( + binding.target, + seen[binding.target], + label, + )) + seen[binding.target] = label + +def _named_port_target(label, name): + """Returns the port target label for a named port owned by `label` (e.g. `@@//pkg:svc.http`). + + This is the analysis-phase counterpart to `_to_relative_named_port` in `//:itest.bzl`, which can + only run during loading. Both must agree on the `