diff --git a/README.md b/README.md index 6ecebcae..e37085ea 100644 --- a/README.md +++ b/README.md @@ -83,6 +83,11 @@ provides functions for - hooking the plugin into pod/container lifecycle events - shutting down the plugin +An additional interface is provided for validating the changes active plugins +have requested to containers. This interface allows one to set up and enforce +cluster- or node-wide boundary conditions for changes NRI plugins are allowed +to make. + ### Plugin Registration Before a plugin can start receiving and processing container events, it needs @@ -277,6 +282,80 @@ can be updated this way: - Block I/O class - RDT class +### Container Adjustment Validation + +NRI plugins operate as trusted extensions of the container runtime, granting +them significant privileges to alter container specs. While this extensibility +is powerful with valid use cases, some of the capabilities granted to plugins +allow modifying security-sensitive settings of containers. As such they also +come with the risk that a plugin could inadvertently or maliciously weaken a +container's isolation or security posture, potentially overriding policies set +by cluster orchestrators such as K8s. + +NRI offers cluster administrators a mechanism to exercise fine-grained control +over what changes plugins are allowed to make to containers, allowing cluster +administrators to lock down selected features in NRI or allowing them to only +be used a subset of plugins. Changes in NRI are made in two phases: “Mutating” +plugins propose changes, and “Validating” plugins approve or deny them. + +Validating plugins are invoked during container creation after all the changes +requested to containers have been collected. Validating plugins receive the +changes with extra information about which of the plugins requested what +changes. They can then choose to reject the changes if they violate some of the +conditions being validated. + +Validation has transactional semantics. If any validating plugin rejects an +adjustment, creation of the adjusted container will fail and none of the other +related changes will be made. + +#### Validation Use Cases + +Some key validation uses cases include + +1. Functional Validators: These plugins care about the final state and +consistency. They check if the combined effect of all mutations result +in a valid configuration (e.g. are the resource limits sane). + +2. Security Validators: These plugins are interested in which plugin is +attempting to modify sensitive fields. They use the extra data passed to +plugins in addition to adjustments to check if a potentially untrusted +plugin tried to modify a restricted field, regardless of the value. +Rejection might occur simply because a non-approved plugin touched a +specific field. Plugins like this may need to be assured to run, and to +have workloads fail-closed if the validator is not running. + +3. Mandatory Plugin Validators: These ensure that specific plugins, required +for certain workloads have successfully run. They might use the extra metadata +passed to validator in addition to adjustments to confirm the mandatory +plugin owns certain critical fields and potentially use the list of plugins +that processed the container to ensure all mandatory plugins were consulted. + +#### Default Validation + +The default built-in validator plugin provides configurable minimal validation. +It is disabled by default. It can be enabled and selectively configured to + +1. Reject OCI Hook injection: Reject any adjustment which tries to inject +OCI Hooks into a container. + +2. Verify global mandatory plugins: Verify that all configured mandatory +plugins are present and have processed a container. Otherwise reject the +creation of the container. + +3. Verify annotated mandatory plugins: Verify that an annotated set of +container-specific mandatory plugins are present and have processed a +container. Otherwise reject the creation of the container. + +Containers can be annotated to tolerate missing required plugins. This +allows one to deploy mandatory plugins as containers themselves. + +#### Default Validation Scope + +Currently only OCI hook injection can be restricted using the default +validator. However, this probably will change in the future. Especially +when NRI is extended with control over new container parameters. If such +parameters will have security implications, corresponding configurable +restrictions will be introduced to the default validator. ## Runtime Adaptation diff --git a/go.mod b/go.mod index 369346c6..5f29dfb9 100644 --- a/go.mod +++ b/go.mod @@ -17,6 +17,7 @@ require ( golang.org/x/sys v0.21.0 google.golang.org/grpc v1.57.1 google.golang.org/protobuf v1.34.1 + gopkg.in/yaml.v3 v3.0.1 ) require ( @@ -35,7 +36,6 @@ require ( golang.org/x/tools v0.21.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20230731190214-cbb8c96f2d6d // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect ) replace github.com/opencontainers/runtime-tools v0.9.0 => github.com/opencontainers/runtime-tools v0.0.0-20221026201742-946c877fa809 diff --git a/pkg/adaptation/adaptation.go b/pkg/adaptation/adaptation.go index e46c351f..7ae20c12 100644 --- a/pkg/adaptation/adaptation.go +++ b/pkg/adaptation/adaptation.go @@ -27,8 +27,10 @@ import ( "sort" "sync" + "github.com/containerd/nri/pkg/adaptation/builtin" "github.com/containerd/nri/pkg/api" "github.com/containerd/nri/pkg/log" + validator "github.com/containerd/nri/plugins/default-validator/builtin" "github.com/containerd/ttrpc" "github.com/tetratelabs/wazero" "github.com/tetratelabs/wazero/imports/wasi_snapshot_preview1" @@ -70,6 +72,7 @@ type Adaptation struct { listener net.Listener plugins []*plugin validators []*plugin + builtin []*builtin.BuiltinPlugin syncLock sync.RWMutex wasmService *api.PluginPlugin } @@ -123,6 +126,24 @@ func WithTTRPCOptions(clientOpts []ttrpc.ClientOpts, serverOpts []ttrpc.ServerOp } } +// WithBuiltinPlugins sets extra builtin plugins to register. +func WithBuiltinPlugins(plugins ...*builtin.BuiltinPlugin) Option { + return func(r *Adaptation) error { + r.builtin = append(r.builtin, plugins...) + return nil + } +} + +// WithDefaultValidator sets up builtin validator plugin if it is configured. +func WithDefaultValidator(cfg *validator.DefaultValidatorConfig) Option { + return func(r *Adaptation) error { + if plugin := validator.GetDefaultValidator(cfg); plugin != nil { + r.builtin = append([]*builtin.BuiltinPlugin{plugin}, r.builtin...) + } + return nil + } +} + // New creates a new NRI Runtime. func New(name, version string, syncFn SyncFn, updateFn UpdateFn, opts ...Option) (*Adaptation, error) { var err error @@ -433,6 +454,20 @@ func (r *Adaptation) startPlugins() (retErr error) { } }() + for _, b := range r.builtin { + log.Infof(noCtx, "starting builtin NRI plugin %q...", b.Index+"-"+b.Base) + p, err := r.newBuiltinPlugin(b) + if err != nil { + return fmt.Errorf("failed to initialize builtin NRI plugin %q: %v", b.Base, err) + } + + if err := p.start(r.name, r.version); err != nil { + return fmt.Errorf("failed to start builtin NRI plugin %q: %v", b.Base, err) + } + + plugins = append(plugins, p) + } + for i, name := range names { log.Infof(noCtx, "starting pre-installed NRI plugin %q...", name) diff --git a/pkg/adaptation/adaptation_suite_test.go b/pkg/adaptation/adaptation_suite_test.go index 6dfcfb02..8d4f784a 100644 --- a/pkg/adaptation/adaptation_suite_test.go +++ b/pkg/adaptation/adaptation_suite_test.go @@ -36,6 +36,8 @@ import ( nri "github.com/containerd/nri/pkg/adaptation" "github.com/containerd/nri/pkg/api" + "github.com/containerd/nri/pkg/plugin" + validator "github.com/containerd/nri/plugins/default-validator/builtin" ) var _ = Describe("Configuration", func() { @@ -1015,6 +1017,275 @@ var _ = Describe("Plugin container creation adjustments", func() { ) }) + When("the default validator is enabled and OCI Hook injection is disabled", func() { + BeforeEach(func() { + s.Prepare( + &mockRuntime{ + options: []nri.Option{ + nri.WithDefaultValidator( + &validator.DefaultValidatorConfig{ + Enable: true, + RejectOCIHooks: true, + }, + ), + }, + }, + &mockPlugin{idx: "00", name: "foo"}, + &mockPlugin{idx: "10", name: "validator1"}, + &mockPlugin{idx: "20", name: "validator2"}, + ) + }) + + It("should reject OCI Hook injection", func() { + var ( + create = func(_ *mockPlugin, _ *api.PodSandbox, ctr *api.Container) (*api.ContainerAdjustment, []*api.ContainerUpdate, error) { + a := &api.ContainerAdjustment{} + if ctr.GetName() == "ctr1" { + a.AddHooks( + &api.Hooks{ + Prestart: []*api.Hook{ + { + Path: "/bin/sh", + Args: []string{"/bin/sh", "-c", "true"}, + }, + }, + }, + ) + } + + return a, nil, nil + } + + validate = func(_ *mockPlugin, _ *api.ValidateContainerAdjustmentRequest) error { + return nil + } + + runtime = s.runtime + plugins = s.plugins + ctx = context.Background() + + pod = &api.PodSandbox{ + Id: "pod0", + Name: "pod0", + Uid: "uid0", + Namespace: "default", + } + ctr0 = &api.Container{ + Id: "ctr0", + PodSandboxId: "pod0", + Name: "ctr0", + State: api.ContainerState_CONTAINER_CREATED, + } + ctr1 = &api.Container{ + Id: "ctr1", + PodSandboxId: "pod0", + Name: "ctr1", + State: api.ContainerState_CONTAINER_CREATED, + } + ) + + plugins[0].createContainer = create + plugins[1].validateAdjustment = validate + plugins[2].validateAdjustment = validate + + s.Startup() + podReq := &api.RunPodSandboxRequest{Pod: pod} + Expect(runtime.RunPodSandbox(ctx, podReq)).To(Succeed()) + + ctrReq := &api.CreateContainerRequest{ + Pod: pod, + Container: ctr0, + } + reply, err := runtime.CreateContainer(ctx, ctrReq) + Expect(reply).ToNot(BeNil()) + Expect(err).To(BeNil()) + + ctrReq = &api.CreateContainerRequest{ + Pod: pod, + Container: ctr1, + } + reply, err = runtime.CreateContainer(ctx, ctrReq) + Expect(err).ToNot(BeNil()) + Expect(reply).To(BeNil()) + }) + }) + + When("the default validator is enabled with some required plugins", func() { + const AnnotationDomain = plugin.AnnotationDomain + BeforeEach(func() { + s.Prepare( + &mockRuntime{ + options: []nri.Option{ + nri.WithDefaultValidator( + &validator.DefaultValidatorConfig{ + Enable: true, + RequiredPlugins: []string{ + "foo", + "bar", + }, + TolerateMissingAnnotation: "tolerate-missing-plugins." + AnnotationDomain, + }, + ), + }, + }, + &mockPlugin{idx: "00", name: "foo"}, + ) + }) + + It("should not allow container creation if required plugins are missing", func() { + var ( + runtime = s.runtime + ctx = context.Background() + + pod = &api.PodSandbox{ + Id: "pod0", + Name: "pod0", + Uid: "uid0", + Namespace: "default", + } + ) + + s.Startup() + podReq := &api.RunPodSandboxRequest{Pod: pod} + Expect(runtime.RunPodSandbox(ctx, podReq)).To(Succeed()) + + ctrReq := &api.CreateContainerRequest{ + Pod: pod, + Container: &api.Container{ + Id: "ctr0", + PodSandboxId: "pod0", + Name: "ctr0", + State: api.ContainerState_CONTAINER_CREATED, + }, + } + reply, err := runtime.CreateContainer(ctx, ctrReq) + Expect(reply).To(BeNil()) + Expect(err).ToNot(BeNil()) + }) + + It("should allow container creation, if missing plugins are tolerated", func() { + var ( + runtime = s.runtime + ctx = context.Background() + + pod = &api.PodSandbox{ + Id: "pod0", + Name: "pod0", + Uid: "uid0", + Namespace: "default", + Annotations: map[string]string{ + "tolerate-missing-plugins." + AnnotationDomain + "/container.ctr0": "true", + }, + } + ) + + s.Startup() + podReq := &api.RunPodSandboxRequest{Pod: pod} + Expect(runtime.RunPodSandbox(ctx, podReq)).To(Succeed()) + + ctrReq := &api.CreateContainerRequest{ + Pod: pod, + Container: &api.Container{ + Id: "ctr0", + PodSandboxId: "pod0", + Name: "ctr0", + State: api.ContainerState_CONTAINER_CREATED, + }, + } + reply, err := runtime.CreateContainer(ctx, ctrReq) + Expect(reply).ToNot(BeNil()) + Expect(err).To(BeNil()) + }) + + It("should allow container creation if all required plugins are present", func() { + var ( + runtime = s.runtime + ctx = context.Background() + + pod = &api.PodSandbox{ + Id: "pod0", + Name: "pod0", + Uid: "uid0", + Namespace: "default", + } + ) + + s.Startup() + podReq := &api.RunPodSandboxRequest{Pod: pod} + Expect(runtime.RunPodSandbox(ctx, podReq)).To(Succeed()) + + s.StartPlugins(&mockPlugin{idx: "10", name: "bar"}) + s.WaitForPluginsToSync(s.plugin("10-bar")) + + ctrReq := &api.CreateContainerRequest{ + Pod: pod, + Container: &api.Container{ + Id: "ctr0", + PodSandboxId: "pod0", + Name: "ctr0", + State: api.ContainerState_CONTAINER_CREATED, + }, + } + reply, err := runtime.CreateContainer(ctx, ctrReq) + Expect(reply).ToNot(BeNil()) + Expect(err).To(BeNil()) + }) + + It("should not allow container creation if annotated required plugins are missing", func() { + var ( + runtime = s.runtime + ctx = context.Background() + + pod = &api.PodSandbox{ + Id: "pod0", + Name: "pod0", + Uid: "uid0", + Namespace: "default", + Annotations: map[string]string{ + "required-plugins." + AnnotationDomain + "/container.ctr0": "[ \"xyzzy\" ]", + }, + } + ) + + s.Startup() + podReq := &api.RunPodSandboxRequest{Pod: pod} + Expect(runtime.RunPodSandbox(ctx, podReq)).To(Succeed()) + + s.StartPlugins(&mockPlugin{idx: "10", name: "bar"}) + s.WaitForPluginsToSync(s.plugin("10-bar")) + + ctrReq := &api.CreateContainerRequest{ + Pod: pod, + Container: &api.Container{ + Id: "ctr0", + PodSandboxId: "pod0", + Name: "ctr0", + State: api.ContainerState_CONTAINER_CREATED, + }, + } + reply, err := runtime.CreateContainer(ctx, ctrReq) + Expect(reply).To(BeNil()) + Expect(err).ToNot(BeNil()) + + s.StartPlugins(&mockPlugin{idx: "20", name: "xyzzy"}) + s.WaitForPluginsToSync(s.plugin("20-xyzzy")) + + ctrReq = &api.CreateContainerRequest{ + Pod: pod, + Container: &api.Container{ + Id: "ctr0", + PodSandboxId: "pod0", + Name: "ctr0", + State: api.ContainerState_CONTAINER_CREATED, + }, + } + reply, err = runtime.CreateContainer(ctx, ctrReq) + Expect(reply).ToNot(BeNil()) + Expect(err).To(BeNil()) + }) + + }) + }) // -------------------------------------------- diff --git a/pkg/adaptation/api.go b/pkg/adaptation/api.go index eeab0887..e4a432c0 100644 --- a/pkg/adaptation/api.go +++ b/pkg/adaptation/api.go @@ -37,6 +37,9 @@ type ( SynchronizeRequest = api.SynchronizeRequest SynchronizeResponse = api.SynchronizeResponse + ShutdownRequest = api.Empty + ShutdownResponse = api.Empty + CreateContainerRequest = api.CreateContainerRequest CreateContainerResponse = api.CreateContainerResponse UpdateContainerRequest = api.UpdateContainerRequest diff --git a/pkg/adaptation/builtin/plugin.go b/pkg/adaptation/builtin/plugin.go new file mode 100644 index 00000000..d818c182 --- /dev/null +++ b/pkg/adaptation/builtin/plugin.go @@ -0,0 +1,210 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package builtin + +import ( + "context" + + "github.com/containerd/nri/pkg/api" +) + +type BuiltinPlugin struct { + Base string + Index string + Handlers BuiltinHandlers +} + +type BuiltinHandlers struct { + Configure func(context.Context, *api.ConfigureRequest) (*api.ConfigureResponse, error) + Synchronize func(context.Context, *api.SynchronizeRequest) (*api.SynchronizeResponse, error) + RunPodSandbox func(context.Context, *api.RunPodSandboxRequest) error + StopPodSandbox func(context.Context, *api.StopPodSandboxRequest) error + RemovePodSandbox func(context.Context, *api.RemovePodSandboxRequest) error + UpdatePodSandbox func(context.Context, *api.UpdatePodSandboxRequest) (*api.UpdatePodSandboxResponse, error) + PostUpdatePodSandbox func(context.Context, *api.PostUpdatePodSandboxRequest) error + + CreateContainer func(context.Context, *api.CreateContainerRequest) (*api.CreateContainerResponse, error) + PostCreateContainer func(context.Context, *api.PostCreateContainerRequest) error + StartContainer func(context.Context, *api.StartContainerRequest) error + PostStartContainer func(context.Context, *api.PostStartContainerRequest) error + UpdateContainer func(context.Context, *api.UpdateContainerRequest) (*api.UpdateContainerResponse, error) + PostUpdateContainer func(context.Context, *api.PostUpdateContainerRequest) error + StopContainer func(context.Context, *api.StopContainerRequest) (*api.StopContainerResponse, error) + RemoveContainer func(context.Context, *api.RemoveContainerRequest) error + ValidateContainerAdjustment func(context.Context, *api.ValidateContainerAdjustmentRequest) error +} + +func (b *BuiltinPlugin) Configure(ctx context.Context, req *api.ConfigureRequest) (*api.ConfigureResponse, error) { + var ( + rpl = &api.ConfigureResponse{} + err error + ) + + if b.Handlers.Configure != nil { + rpl, err = b.Handlers.Configure(ctx, req) + } + + if rpl.Events == 0 { + var events api.EventMask + + if b.Handlers.RunPodSandbox != nil { + events.Set(api.Event_RUN_POD_SANDBOX) + } + if b.Handlers.StopPodSandbox != nil { + events.Set(api.Event_STOP_POD_SANDBOX) + } + if b.Handlers.RemovePodSandbox != nil { + events.Set(api.Event_REMOVE_POD_SANDBOX) + } + if b.Handlers.UpdatePodSandbox != nil { + events.Set(api.Event_UPDATE_POD_SANDBOX) + } + if b.Handlers.PostUpdatePodSandbox != nil { + events.Set(api.Event_POST_UPDATE_POD_SANDBOX) + } + if b.Handlers.CreateContainer != nil { + events.Set(api.Event_CREATE_CONTAINER) + } + if b.Handlers.PostCreateContainer != nil { + events.Set(api.Event_POST_CREATE_CONTAINER) + } + if b.Handlers.StartContainer != nil { + events.Set(api.Event_START_CONTAINER) + } + if b.Handlers.PostStartContainer != nil { + events.Set(api.Event_POST_START_CONTAINER) + } + if b.Handlers.UpdateContainer != nil { + events.Set(api.Event_UPDATE_CONTAINER) + } + if b.Handlers.PostUpdateContainer != nil { + events.Set(api.Event_POST_UPDATE_CONTAINER) + } + if b.Handlers.StopContainer != nil { + events.Set(api.Event_STOP_CONTAINER) + } + if b.Handlers.RemoveContainer != nil { + events.Set(api.Event_REMOVE_CONTAINER) + } + if b.Handlers.ValidateContainerAdjustment != nil { + events.Set(api.Event_VALIDATE_CONTAINER_ADJUSTMENT) + } + + rpl.Events = int32(events) + } + + return rpl, err +} + +func (b *BuiltinPlugin) Synchronize(ctx context.Context, req *api.SynchronizeRequest) (*api.SynchronizeResponse, error) { + if b.Handlers.Synchronize != nil { + return b.Handlers.Synchronize(ctx, req) + } + return &api.SynchronizeResponse{}, nil +} + +func (b *BuiltinPlugin) Shutdown(context.Context, *api.ShutdownRequest) (*api.ShutdownResponse, error) { + return &api.ShutdownResponse{}, nil +} + +func (b *BuiltinPlugin) CreateContainer(ctx context.Context, req *api.CreateContainerRequest) (*api.CreateContainerResponse, error) { + if b.Handlers.CreateContainer != nil { + return b.Handlers.CreateContainer(ctx, req) + } + return &api.CreateContainerResponse{}, nil +} + +func (b *BuiltinPlugin) UpdateContainer(ctx context.Context, req *api.UpdateContainerRequest) (*api.UpdateContainerResponse, error) { + if b.Handlers.UpdateContainer != nil { + return b.Handlers.UpdateContainer(ctx, req) + } + return &api.UpdateContainerResponse{}, nil +} + +func (b *BuiltinPlugin) StopContainer(ctx context.Context, req *api.StopContainerRequest) (*api.StopContainerResponse, error) { + if b.Handlers.StopContainer != nil { + return b.Handlers.StopContainer(ctx, req) + } + return &api.StopContainerResponse{}, nil +} + +func (b *BuiltinPlugin) StateChange(ctx context.Context, evt *api.StateChangeEvent) (*api.StateChangeResponse, error) { + var err error + switch evt.Event { + case api.Event_RUN_POD_SANDBOX: + if b.Handlers.RunPodSandbox != nil { + err = b.Handlers.RunPodSandbox(ctx, evt) + } + case api.Event_STOP_POD_SANDBOX: + if b.Handlers.StopPodSandbox != nil { + err = b.Handlers.StopPodSandbox(ctx, evt) + } + case api.Event_REMOVE_POD_SANDBOX: + if b.Handlers.RemovePodSandbox != nil { + err = b.Handlers.RemovePodSandbox(ctx, evt) + } + case api.Event_POST_CREATE_CONTAINER: + if b.Handlers.PostCreateContainer != nil { + err = b.Handlers.PostCreateContainer(ctx, evt) + } + case api.Event_START_CONTAINER: + if b.Handlers.StartContainer != nil { + err = b.Handlers.StartContainer(ctx, evt) + } + case api.Event_POST_START_CONTAINER: + if b.Handlers.PostStartContainer != nil { + err = b.Handlers.PostStartContainer(ctx, evt) + } + case api.Event_POST_UPDATE_CONTAINER: + if b.Handlers.PostUpdateContainer != nil { + err = b.Handlers.PostUpdateContainer(ctx, evt) + } + case api.Event_REMOVE_CONTAINER: + if b.Handlers.RemoveContainer != nil { + err = b.Handlers.RemoveContainer(ctx, evt) + } + } + + return &api.StateChangeResponse{}, err +} + +func (b *BuiltinPlugin) UpdatePodSandbox(ctx context.Context, req *api.UpdatePodSandboxRequest) (*api.UpdatePodSandboxResponse, error) { + if b.Handlers.UpdatePodSandbox != nil { + return b.Handlers.UpdatePodSandbox(ctx, req) + } + return &api.UpdatePodSandboxResponse{}, nil +} + +func (b *BuiltinPlugin) PostUpdatePodSandbox(ctx context.Context, req *api.PostUpdatePodSandboxRequest) error { + if b.Handlers.PostUpdatePodSandbox != nil { + return b.Handlers.PostUpdatePodSandbox(ctx, req) + } + return nil +} + +func (b *BuiltinPlugin) ValidateContainerAdjustment(ctx context.Context, req *api.ValidateContainerAdjustmentRequest) (*api.ValidateContainerAdjustmentResponse, error) { + if b.Handlers.ValidateContainerAdjustment != nil { + if err := b.Handlers.ValidateContainerAdjustment(ctx, req); err != nil { + return &api.ValidateContainerAdjustmentResponse{ + Reject: true, + Reason: err.Error(), + }, nil + } + } + + return &api.ValidateContainerAdjustmentResponse{}, nil +} diff --git a/pkg/adaptation/plugin.go b/pkg/adaptation/plugin.go index 7954b864..80e5555b 100644 --- a/pkg/adaptation/plugin.go +++ b/pkg/adaptation/plugin.go @@ -28,6 +28,7 @@ import ( "sync" "time" + "github.com/containerd/nri/pkg/adaptation/builtin" "github.com/containerd/nri/pkg/api" "github.com/containerd/nri/pkg/log" "github.com/containerd/nri/pkg/net" @@ -165,6 +166,20 @@ func (r *Adaptation) newLaunchedPlugin(dir, idx, base, cfg string) (p *plugin, r return p, nil } +func (r *Adaptation) newBuiltinPlugin(b *builtin.BuiltinPlugin) (*plugin, error) { + if b.Base == "" || b.Index == "" { + return nil, fmt.Errorf("builtin plugin without index or name (%q, %q)", b.Index, b.Base) + } + + return &plugin{ + idx: b.Index, + base: b.Base, + closeC: make(chan struct{}), + r: r, + impl: &pluginType{builtinImpl: b}, + }, nil +} + func isWasm(path string) bool { file, err := os.Open(path) if err != nil { @@ -296,7 +311,7 @@ func (p *plugin) connect(conn stdnet.Conn) (retErr error) { // Start Runtime service, wait for plugin to register, then configure it. func (p *plugin) start(name, version string) (err error) { - // skip start for WASM plugins and head right to the registration for + // skip start for WASM and builtin plugins and head right to the registration for // events config if p.impl.isTtrpc() { var ( @@ -340,7 +355,7 @@ func (p *plugin) start(name, version string) (err error) { // close a plugin shutting down its multiplexed ttrpc connections. func (p *plugin) close() { - if p.impl.isWasm() { + if p.impl.isWasm() || p.impl.isBuiltin() { p.closed = true return } @@ -366,7 +381,7 @@ func (p *plugin) isClosed() bool { // stop a plugin (if it was launched by us) func (p *plugin) stop() error { - if p.isExternal() || p.cmd.Process == nil || p.impl.isWasm() { + if p.isExternal() || p.cmd.Process == nil || p.impl.isWasm() || p.impl.isBuiltin() { return nil } @@ -389,11 +404,20 @@ func (p *plugin) name() string { } func (p *plugin) qualifiedName() string { - var kind, idx, base string - if p.isExternal() { - kind = "external" + var kind, idx, base, pid string + if p.impl.isBuiltin() { + kind = "builtin" } else { - kind = "pre-connected" + if p.isExternal() { + kind = "external" + } else { + kind = "pre-connected" + } + if p.impl.isWasm() { + kind += "-wasm" + } else { + pid = "[" + strconv.Itoa(p.pid) + "]" + } } if idx = p.idx; idx == "" { idx = "??" @@ -401,7 +425,7 @@ func (p *plugin) qualifiedName() string { if base = p.base; base == "" { base = "plugin" } - return kind + ":" + idx + "-" + base + "[" + strconv.Itoa(p.pid) + "]" + return kind + ":" + idx + "-" + base + pid } // RegisterPlugin handles the plugin's registration request. diff --git a/pkg/adaptation/plugin_type.go b/pkg/adaptation/plugin_type.go index cfc4c44c..43390291 100644 --- a/pkg/adaptation/plugin_type.go +++ b/pkg/adaptation/plugin_type.go @@ -18,15 +18,21 @@ package adaptation import ( "context" + "errors" "github.com/containerd/nri/pkg/api" ) type pluginType struct { - wasmImpl api.Plugin - ttrpcImpl api.PluginService + wasmImpl api.Plugin + ttrpcImpl api.PluginService + builtinImpl api.PluginService } +var ( + errUnknownImpl = errors.New("unknown plugin implementation type") +) + func (p *pluginType) isWasm() bool { return p.wasmImpl != nil } @@ -35,60 +41,111 @@ func (p *pluginType) isTtrpc() bool { return p.ttrpcImpl != nil } +func (p *pluginType) isBuiltin() bool { + return p.builtinImpl != nil +} + func (p *pluginType) Synchronize(ctx context.Context, req *SynchronizeRequest) (*SynchronizeResponse, error) { - if p.wasmImpl != nil { + switch { + case p.ttrpcImpl != nil: + return p.ttrpcImpl.Synchronize(ctx, req) + case p.builtinImpl != nil: + return p.builtinImpl.Synchronize(ctx, req) + case p.wasmImpl != nil: return p.wasmImpl.Synchronize(ctx, req) } - return p.ttrpcImpl.Synchronize(ctx, req) + + return nil, errUnknownImpl } func (p *pluginType) Configure(ctx context.Context, req *ConfigureRequest) (*ConfigureResponse, error) { - if p.wasmImpl != nil { + switch { + case p.ttrpcImpl != nil: + return p.ttrpcImpl.Configure(ctx, req) + case p.builtinImpl != nil: + return p.builtinImpl.Configure(ctx, req) + case p.wasmImpl != nil: return p.wasmImpl.Configure(ctx, req) } - return p.ttrpcImpl.Configure(ctx, req) + + return nil, errUnknownImpl } func (p *pluginType) CreateContainer(ctx context.Context, req *CreateContainerRequest) (*CreateContainerResponse, error) { - if p.wasmImpl != nil { + switch { + case p.ttrpcImpl != nil: + return p.ttrpcImpl.CreateContainer(ctx, req) + case p.builtinImpl != nil: + return p.builtinImpl.CreateContainer(ctx, req) + case p.wasmImpl != nil: return p.wasmImpl.CreateContainer(ctx, req) } - return p.ttrpcImpl.CreateContainer(ctx, req) + + return nil, errUnknownImpl } func (p *pluginType) UpdateContainer(ctx context.Context, req *UpdateContainerRequest) (*UpdateContainerResponse, error) { - if p.wasmImpl != nil { + switch { + case p.ttrpcImpl != nil: + return p.ttrpcImpl.UpdateContainer(ctx, req) + case p.builtinImpl != nil: + return p.builtinImpl.UpdateContainer(ctx, req) + case p.wasmImpl != nil: return p.wasmImpl.UpdateContainer(ctx, req) } - return p.ttrpcImpl.UpdateContainer(ctx, req) + + return nil, errUnknownImpl } func (p *pluginType) StopContainer(ctx context.Context, req *StopContainerRequest) (*StopContainerResponse, error) { - if p.wasmImpl != nil { + switch { + case p.ttrpcImpl != nil: + return p.ttrpcImpl.StopContainer(ctx, req) + case p.builtinImpl != nil: + return p.builtinImpl.StopContainer(ctx, req) + case p.wasmImpl != nil: return p.wasmImpl.StopContainer(ctx, req) } - return p.ttrpcImpl.StopContainer(ctx, req) + + return nil, errUnknownImpl } func (p *pluginType) UpdatePodSandbox(ctx context.Context, req *UpdatePodSandboxRequest) (*UpdatePodSandboxResponse, error) { - if p.wasmImpl != nil { + switch { + case p.ttrpcImpl != nil: + return p.ttrpcImpl.UpdatePodSandbox(ctx, req) + case p.builtinImpl != nil: + return p.builtinImpl.UpdatePodSandbox(ctx, req) + case p.wasmImpl != nil: return p.wasmImpl.UpdatePodSandbox(ctx, req) } - return p.ttrpcImpl.UpdatePodSandbox(ctx, req) + + return nil, errUnknownImpl } func (p *pluginType) StateChange(ctx context.Context, req *StateChangeEvent) (err error) { - if p.wasmImpl != nil { - _, err = p.wasmImpl.StateChange(ctx, req) - } else { + switch { + case p.ttrpcImpl != nil: _, err = p.ttrpcImpl.StateChange(ctx, req) + case p.builtinImpl != nil: + _, err = p.builtinImpl.StateChange(ctx, req) + case p.wasmImpl != nil: + _, err = p.wasmImpl.StateChange(ctx, req) + default: + err = errUnknownImpl } return err } func (p *pluginType) ValidateContainerAdjustment(ctx context.Context, req *ValidateContainerAdjustmentRequest) (*ValidateContainerAdjustmentResponse, error) { - if p.wasmImpl != nil { + switch { + case p.ttrpcImpl != nil: + return p.ttrpcImpl.ValidateContainerAdjustment(ctx, req) + case p.builtinImpl != nil: + return p.builtinImpl.ValidateContainerAdjustment(ctx, req) + case p.wasmImpl != nil: return p.wasmImpl.ValidateContainerAdjustment(ctx, req) } - return p.ttrpcImpl.ValidateContainerAdjustment(ctx, req) + + return nil, errUnknownImpl } diff --git a/pkg/adaptation/suite_test.go b/pkg/adaptation/suite_test.go index 8021ef6c..885113b9 100644 --- a/pkg/adaptation/suite_test.go +++ b/pkg/adaptation/suite_test.go @@ -53,6 +53,7 @@ type Suite struct { dir string // directory to create for test runtime *mockRuntime // runtime instance for test plugins []*mockPlugin // plugin instances for test + byName map[string]*mockPlugin } // SuiteOption can be applied to a suite. @@ -83,6 +84,10 @@ func (s *Suite) Prepare(runtime *mockRuntime, plugins ...*mockPlugin) string { s.runtime = runtime s.plugins = plugins + if s.byName == nil { + s.byName = make(map[string]*mockPlugin) + } + return dir } @@ -93,9 +98,11 @@ func (s *Suite) Dir() string { // Startup starts up the test suite. func (s *Suite) Startup() { + plugins := s.plugins + s.plugins = nil s.StartRuntime() - s.StartPlugins() - s.WaitForPluginsToSync() + s.StartPlugins(plugins...) + s.WaitForPluginsToSync(plugins...) } // StartRuntime starts the suite runtime. @@ -104,16 +111,18 @@ func (s *Suite) StartRuntime() { } // StartPlugins starts the suite plugins. -func (s *Suite) StartPlugins() { - for _, plugin := range s.plugins { +func (s *Suite) StartPlugins(plugins ...*mockPlugin) { + for _, plugin := range plugins { + s.plugins = append(s.plugins, plugin) + s.byName[plugin.FullName()] = plugin Expect(plugin.Start(s.dir)).To(Succeed()) } } -// WaitForPluginsToSync waits for the suite plugins to get synchronized. -func (s *Suite) WaitForPluginsToSync() { +// WaitForPluginsToSync waits for the given plugins to get synchronized. +func (s *Suite) WaitForPluginsToSync(plugins ...*mockPlugin) { timeout := time.After(startupTimeout) - for _, plugin := range s.plugins { + for _, plugin := range plugins { Expect(plugin.Wait(PluginSynchronized, timeout)).To(Succeed()) } } @@ -128,6 +137,11 @@ func (s *Suite) Cleanup() { Expect(os.RemoveAll(s.dir)).To(Succeed()) } +// Plugin returns a plugin started by StartPlugins by full plugin name. +func (s *Suite) plugin(fullName string) *mockPlugin { + return s.byName[fullName] +} + // ------------------------------------ func Log(format string, args ...interface{}) { @@ -521,6 +535,10 @@ func (m *mockPlugin) Stop() { m.q.Add(PluginStopped) } +func (m *mockPlugin) FullName() string { + return m.idx + "-" + m.name +} + func (m *mockPlugin) RuntimeName() string { return m.runtime } diff --git a/pkg/api/owners.go b/pkg/api/owners.go index e476ac3d..af74c216 100644 --- a/pkg/api/owners.go +++ b/pkg/api/owners.go @@ -38,262 +38,262 @@ func NewOwningPlugins() *OwningPlugins { } func (o *OwningPlugins) ClaimAnnotation(id, key, plugin string) error { - return o.OwnersFor(id).ClaimAnnotation(key, plugin) + return o.mustOwnersFor(id).ClaimAnnotation(key, plugin) } func (o *OwningPlugins) ClaimMount(id, destination, plugin string) error { - return o.OwnersFor(id).ClaimMount(destination, plugin) + return o.mustOwnersFor(id).ClaimMount(destination, plugin) } func (o *OwningPlugins) ClaimHooks(id, plugin string) error { - return o.OwnersFor(id).ClaimHooks(plugin) + return o.mustOwnersFor(id).ClaimHooks(plugin) } func (o *OwningPlugins) ClaimDevice(id, path, plugin string) error { - return o.OwnersFor(id).ClaimDevice(path, plugin) + return o.mustOwnersFor(id).ClaimDevice(path, plugin) } func (o *OwningPlugins) ClaimCdiDevice(id, name, plugin string) error { - return o.OwnersFor(id).ClaimCdiDevice(name, plugin) + return o.mustOwnersFor(id).ClaimCdiDevice(name, plugin) } func (o *OwningPlugins) ClaimEnv(id, name, plugin string) error { - return o.OwnersFor(id).ClaimEnv(name, plugin) + return o.mustOwnersFor(id).ClaimEnv(name, plugin) } func (o *OwningPlugins) ClaimArgs(id, plugin string) error { - return o.OwnersFor(id).ClaimArgs(plugin) + return o.mustOwnersFor(id).ClaimArgs(plugin) } func (o *OwningPlugins) ClaimMemLimit(id, plugin string) error { - return o.OwnersFor(id).ClaimMemLimit(plugin) + return o.mustOwnersFor(id).ClaimMemLimit(plugin) } func (o *OwningPlugins) ClaimMemReservation(id, plugin string) error { - return o.OwnersFor(id).ClaimMemReservation(plugin) + return o.mustOwnersFor(id).ClaimMemReservation(plugin) } func (o *OwningPlugins) ClaimMemSwapLimit(id, plugin string) error { - return o.OwnersFor(id).ClaimMemSwapLimit(plugin) + return o.mustOwnersFor(id).ClaimMemSwapLimit(plugin) } func (o *OwningPlugins) ClaimMemKernelLimit(id, plugin string) error { - return o.OwnersFor(id).ClaimMemKernelLimit(plugin) + return o.mustOwnersFor(id).ClaimMemKernelLimit(plugin) } func (o *OwningPlugins) ClaimMemTCPLimit(id, plugin string) error { - return o.OwnersFor(id).ClaimMemTCPLimit(plugin) + return o.mustOwnersFor(id).ClaimMemTCPLimit(plugin) } func (o *OwningPlugins) ClaimMemSwappiness(id, plugin string) error { - return o.OwnersFor(id).ClaimMemSwappiness(plugin) + return o.mustOwnersFor(id).ClaimMemSwappiness(plugin) } func (o *OwningPlugins) ClaimMemDisableOomKiller(id, plugin string) error { - return o.OwnersFor(id).ClaimMemDisableOomKiller(plugin) + return o.mustOwnersFor(id).ClaimMemDisableOomKiller(plugin) } func (o *OwningPlugins) ClaimMemUseHierarchy(id, plugin string) error { - return o.OwnersFor(id).ClaimMemUseHierarchy(plugin) + return o.mustOwnersFor(id).ClaimMemUseHierarchy(plugin) } func (o *OwningPlugins) ClaimCPUShares(id, plugin string) error { - return o.OwnersFor(id).ClaimCPUShares(plugin) + return o.mustOwnersFor(id).ClaimCPUShares(plugin) } func (o *OwningPlugins) ClaimCPUQuota(id, plugin string) error { - return o.OwnersFor(id).ClaimCPUQuota(plugin) + return o.mustOwnersFor(id).ClaimCPUQuota(plugin) } func (o *OwningPlugins) ClaimCPUPeriod(id, plugin string) error { - return o.OwnersFor(id).ClaimCPUPeriod(plugin) + return o.mustOwnersFor(id).ClaimCPUPeriod(plugin) } func (o *OwningPlugins) ClaimCPURealtimeRuntime(id, plugin string) error { - return o.OwnersFor(id).ClaimCPURealtimeRuntime(plugin) + return o.mustOwnersFor(id).ClaimCPURealtimeRuntime(plugin) } func (o *OwningPlugins) ClaimCPURealtimePeriod(id, plugin string) error { - return o.OwnersFor(id).ClaimCPURealtimePeriod(plugin) + return o.mustOwnersFor(id).ClaimCPURealtimePeriod(plugin) } func (o *OwningPlugins) ClaimCPUSetCPUs(id, plugin string) error { - return o.OwnersFor(id).ClaimCPUSetCPUs(plugin) + return o.mustOwnersFor(id).ClaimCPUSetCPUs(plugin) } func (o *OwningPlugins) ClaimCPUSetMems(id, plugin string) error { - return o.OwnersFor(id).ClaimCPUSetMems(plugin) + return o.mustOwnersFor(id).ClaimCPUSetMems(plugin) } func (o *OwningPlugins) ClaimPidsLimit(id, plugin string) error { - return o.OwnersFor(id).ClaimPidsLimit(plugin) + return o.mustOwnersFor(id).ClaimPidsLimit(plugin) } func (o *OwningPlugins) ClaimHugepageLimit(id, size, plugin string) error { - return o.OwnersFor(id).ClaimHugepageLimit(size, plugin) + return o.mustOwnersFor(id).ClaimHugepageLimit(size, plugin) } func (o *OwningPlugins) ClaimBlockioClass(id, plugin string) error { - return o.OwnersFor(id).ClaimBlockioClass(plugin) + return o.mustOwnersFor(id).ClaimBlockioClass(plugin) } func (o *OwningPlugins) ClaimRdtClass(id, plugin string) error { - return o.OwnersFor(id).ClaimRdtClass(plugin) + return o.mustOwnersFor(id).ClaimRdtClass(plugin) } func (o *OwningPlugins) ClaimCgroupsUnified(id, key, plugin string) error { - return o.OwnersFor(id).ClaimCgroupsUnified(key, plugin) + return o.mustOwnersFor(id).ClaimCgroupsUnified(key, plugin) } func (o *OwningPlugins) ClaimCgroupsPath(id, plugin string) error { - return o.OwnersFor(id).ClaimCgroupsPath(plugin) + return o.mustOwnersFor(id).ClaimCgroupsPath(plugin) } func (o *OwningPlugins) ClaimOomScoreAdj(id, plugin string) error { - return o.OwnersFor(id).ClaimOomScoreAdj(plugin) + return o.mustOwnersFor(id).ClaimOomScoreAdj(plugin) } func (o *OwningPlugins) ClaimRlimit(id, typ, plugin string) error { - return o.OwnersFor(id).ClaimRlimit(typ, plugin) + return o.mustOwnersFor(id).ClaimRlimit(typ, plugin) } func (o *OwningPlugins) ClearAnnotation(id, key, plugin string) { - o.OwnersFor(id).ClearAnnotation(key, plugin) + o.mustOwnersFor(id).ClearAnnotation(key, plugin) } func (o *OwningPlugins) ClearMount(id, key, plugin string) { - o.OwnersFor(id).ClearMount(key, plugin) + o.mustOwnersFor(id).ClearMount(key, plugin) } func (o *OwningPlugins) ClearDevice(id, key, plugin string) { - o.OwnersFor(id).ClearDevice(key, plugin) + o.mustOwnersFor(id).ClearDevice(key, plugin) } func (o *OwningPlugins) ClearEnv(id, key, plugin string) { - o.OwnersFor(id).ClearEnv(key, plugin) + o.mustOwnersFor(id).ClearEnv(key, plugin) } func (o *OwningPlugins) ClearArgs(id, plugin string) { - o.OwnersFor(id).ClearArgs(plugin) + o.mustOwnersFor(id).ClearArgs(plugin) } func (o *OwningPlugins) AnnotationOwner(id, key string) (string, bool) { - return o.OwnersFor(id).compoundOwner(Field_Annotations.Key(), key) + return o.ownersFor(id).compoundOwner(Field_Annotations.Key(), key) } func (o *OwningPlugins) MountOwner(id, destination string) (string, bool) { - return o.OwnersFor(id).compoundOwner(Field_Mounts.Key(), destination) + return o.ownersFor(id).compoundOwner(Field_Mounts.Key(), destination) } func (o *OwningPlugins) HooksOwner(id string) (string, bool) { - return o.OwnersFor(id).simpleOwner(Field_OciHooks.Key()) + return o.ownersFor(id).simpleOwner(Field_OciHooks.Key()) } func (o *OwningPlugins) DeviceOwner(id, path string) (string, bool) { - return o.OwnersFor(id).compoundOwner(Field_Devices.Key(), path) + return o.ownersFor(id).compoundOwner(Field_Devices.Key(), path) } func (o *OwningPlugins) EnvOwner(id, name string) (string, bool) { - return o.OwnersFor(id).compoundOwner(Field_Env.Key(), name) + return o.ownersFor(id).compoundOwner(Field_Env.Key(), name) } func (o *OwningPlugins) ArgsOwner(id string) (string, bool) { - return o.OwnersFor(id).simpleOwner(Field_Args.Key()) + return o.ownersFor(id).simpleOwner(Field_Args.Key()) } func (o *OwningPlugins) MemLimitOwner(id string) (string, bool) { - return o.OwnersFor(id).simpleOwner(Field_MemLimit.Key()) + return o.ownersFor(id).simpleOwner(Field_MemLimit.Key()) } func (o *OwningPlugins) MemReservationOwner(id string) (string, bool) { - return o.OwnersFor(id).simpleOwner(Field_MemReservation.Key()) + return o.ownersFor(id).simpleOwner(Field_MemReservation.Key()) } func (o *OwningPlugins) MemSwapLimitOwner(id string) (string, bool) { - return o.OwnersFor(id).simpleOwner(Field_MemSwapLimit.Key()) + return o.ownersFor(id).simpleOwner(Field_MemSwapLimit.Key()) } func (o *OwningPlugins) MemKernelLimitOwner(id string) (string, bool) { - return o.OwnersFor(id).simpleOwner(Field_MemKernelLimit.Key()) + return o.ownersFor(id).simpleOwner(Field_MemKernelLimit.Key()) } func (o *OwningPlugins) MemTCPLimitOwner(id string) (string, bool) { - return o.OwnersFor(id).simpleOwner(Field_MemTCPLimit.Key()) + return o.ownersFor(id).simpleOwner(Field_MemTCPLimit.Key()) } func (o *OwningPlugins) MemSwappinessOwner(id string) (string, bool) { - return o.OwnersFor(id).simpleOwner(Field_MemSwappiness.Key()) + return o.ownersFor(id).simpleOwner(Field_MemSwappiness.Key()) } func (o *OwningPlugins) MemDisableOomKillerOwner(id string) (string, bool) { - return o.OwnersFor(id).simpleOwner(Field_MemDisableOomKiller.Key()) + return o.ownersFor(id).simpleOwner(Field_MemDisableOomKiller.Key()) } func (o *OwningPlugins) MemUseHierarchyOwner(id string) (string, bool) { - return o.OwnersFor(id).simpleOwner(Field_MemUseHierarchy.Key()) + return o.ownersFor(id).simpleOwner(Field_MemUseHierarchy.Key()) } func (o *OwningPlugins) CPUSharesOwner(id string) (string, bool) { - return o.OwnersFor(id).simpleOwner(Field_CPUShares.Key()) + return o.ownersFor(id).simpleOwner(Field_CPUShares.Key()) } func (o *OwningPlugins) CPUQuotaOwner(id string) (string, bool) { - return o.OwnersFor(id).simpleOwner(Field_CPUQuota.Key()) + return o.ownersFor(id).simpleOwner(Field_CPUQuota.Key()) } func (o *OwningPlugins) CPUPeriodOwner(id string) (string, bool) { - return o.OwnersFor(id).simpleOwner(Field_CPUPeriod.Key()) + return o.ownersFor(id).simpleOwner(Field_CPUPeriod.Key()) } func (o *OwningPlugins) CPURealtimeRuntimeOwner(id string) (string, bool) { - return o.OwnersFor(id).simpleOwner(Field_CPURealtimeRuntime.Key()) + return o.ownersFor(id).simpleOwner(Field_CPURealtimeRuntime.Key()) } func (o *OwningPlugins) CPURealtimePeriodOwner(id string) (string, bool) { - return o.OwnersFor(id).simpleOwner(Field_CPURealtimePeriod.Key()) + return o.ownersFor(id).simpleOwner(Field_CPURealtimePeriod.Key()) } func (o *OwningPlugins) CPUSetCPUsOwner(id string) (string, bool) { - return o.OwnersFor(id).simpleOwner(Field_CPUSetCPUs.Key()) + return o.ownersFor(id).simpleOwner(Field_CPUSetCPUs.Key()) } func (o *OwningPlugins) CPUSetMemsOwner(id string) (string, bool) { - return o.OwnersFor(id).simpleOwner(Field_CPUSetMems.Key()) + return o.ownersFor(id).simpleOwner(Field_CPUSetMems.Key()) } func (o *OwningPlugins) PidsLimitOwner(id string) (string, bool) { - return o.OwnersFor(id).simpleOwner(Field_PidsLimit.Key()) + return o.ownersFor(id).simpleOwner(Field_PidsLimit.Key()) } func (o *OwningPlugins) HugepageLimitOwner(id, size string) (string, bool) { - return o.OwnersFor(id).compoundOwner(Field_HugepageLimits.Key(), size) + return o.ownersFor(id).compoundOwner(Field_HugepageLimits.Key(), size) } func (o *OwningPlugins) BlockioClassOwner(id string) (string, bool) { - return o.OwnersFor(id).simpleOwner(Field_BlockioClass.Key()) + return o.ownersFor(id).simpleOwner(Field_BlockioClass.Key()) } func (o *OwningPlugins) RdtClassOwner(id string) (string, bool) { - return o.OwnersFor(id).simpleOwner(Field_RdtClass.Key()) + return o.ownersFor(id).simpleOwner(Field_RdtClass.Key()) } func (o *OwningPlugins) CgroupsUnifiedOwner(id, key string) (string, bool) { - return o.OwnersFor(id).compoundOwner(Field_CgroupsUnified.Key(), key) + return o.ownersFor(id).compoundOwner(Field_CgroupsUnified.Key(), key) } func (o *OwningPlugins) CgroupsPathOwner(id string) (string, bool) { - return o.OwnersFor(id).simpleOwner(Field_CgroupsPath.Key()) + return o.ownersFor(id).simpleOwner(Field_CgroupsPath.Key()) } func (o *OwningPlugins) OomScoreAdjOwner(id string) (string, bool) { - return o.OwnersFor(id).simpleOwner(Field_OomScoreAdj.Key()) + return o.ownersFor(id).simpleOwner(Field_OomScoreAdj.Key()) } func (o *OwningPlugins) RlimitOwner(id, typ string) (string, bool) { - return o.OwnersFor(id).compoundOwner(Field_Rlimits.Key(), typ) + return o.ownersFor(id).compoundOwner(Field_Rlimits.Key(), typ) } -func (o *OwningPlugins) OwnersFor(id string) *FieldOwners { +func (o *OwningPlugins) mustOwnersFor(id string) *FieldOwners { f, ok := o.Owners[id] if !ok { f = NewFieldOwners() @@ -302,6 +302,14 @@ func (o *OwningPlugins) OwnersFor(id string) *FieldOwners { return f } +func (o *OwningPlugins) ownersFor(id string) *FieldOwners { + f, ok := o.Owners[id] + if !ok { + return nil + } + return f +} + func NewFieldOwners() *FieldOwners { return &FieldOwners{ Simple: make(map[int32]string), diff --git a/pkg/api/validate.go b/pkg/api/validate.go index 1d636dda..aa7d68c8 100644 --- a/pkg/api/validate.go +++ b/pkg/api/validate.go @@ -48,3 +48,17 @@ func (v *ValidateContainerAdjustmentResponse) ValidationResult(plugin string) er return fmt.Errorf("validator %q rejected container adjustment, reason: %s", plugin, reason) } + +func (v *ValidateContainerAdjustmentRequest) GetPluginMap() map[string]*PluginInstance { + if v == nil { + return nil + } + + plugins := make(map[string]*PluginInstance) + for _, p := range v.Plugins { + plugins[p.Name] = &PluginInstance{Name: p.Name} + plugins[p.Index+"-"+p.Name] = p + } + + return plugins +} diff --git a/pkg/plugin/annotations.go b/pkg/plugin/annotations.go new file mode 100644 index 00000000..45160941 --- /dev/null +++ b/pkg/plugin/annotations.go @@ -0,0 +1,59 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package plugin + +import ( + "github.com/containerd/nri/pkg/api" +) + +const ( + // AnnotationDomain is the domain used for NRI-specific annotations. + AnnotationDomain = "noderesource.dev" + + // RequiredPluginsAnnotation can be used to annotate pods with a list + // of pod- or container-specific plugins which must process containers + // during creation. If enabled, the default validator checks for this + // and rejects the creation of containers which fail this check. + RequiredPluginsAnnotation = "required-plugins." + AnnotationDomain +) + +// GetEffectiveAnnotation retrieves a custom annotation from a pod which +// applies to given container. The syntax allows both pod- and container- +// scoped annotations. Container-scoped annotations take precedence over +// pod-scoped ones. The key syntax defines the scope of the annotation. +// - container-scope: /container. +// - pod-scope: /pod, or just +func GetEffectiveAnnotation(pod *api.PodSandbox, key, container string) (string, bool) { + annotations := pod.GetAnnotations() + if len(annotations) == 0 { + return "", false + } + + keys := []string{ + key + "/container." + container, + key + "/pod", + key, + } + + for _, k := range keys { + if v, ok := annotations[k]; ok { + return v, true + } + } + + return "", false +} diff --git a/plugins/default-validator/builtin/plugin.go b/plugins/default-validator/builtin/plugin.go new file mode 100644 index 00000000..d391bbcc --- /dev/null +++ b/plugins/default-validator/builtin/plugin.go @@ -0,0 +1,48 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package builtin + +import ( + "context" + + "github.com/containerd/nri/pkg/adaptation/builtin" + "github.com/containerd/nri/pkg/log" + + validator "github.com/containerd/nri/plugins/default-validator" +) + +type ( + DefaultValidatorConfig = validator.DefaultValidatorConfig +) + +// GetDefaultValidator returns a configured instance of the default validator. +// If default validation is disabled nil is returned. +func GetDefaultValidator(cfg *DefaultValidatorConfig) *builtin.BuiltinPlugin { + if cfg == nil || !cfg.Enable { + log.Infof(context.TODO(), "built-in NRI default validator is disabled") + return nil + } + + v := validator.NewDefaultValidator(cfg) + return &builtin.BuiltinPlugin{ + Base: "default-validator", + Index: "00", + Handlers: builtin.BuiltinHandlers{ + ValidateContainerAdjustment: v.ValidateContainerAdjustment, + }, + } +} diff --git a/plugins/default-validator/default-validator.go b/plugins/default-validator/default-validator.go new file mode 100644 index 00000000..cc9d404a --- /dev/null +++ b/plugins/default-validator/default-validator.go @@ -0,0 +1,177 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package validator + +import ( + "context" + "errors" + "fmt" + "slices" + "strconv" + "strings" + + "github.com/containerd/nri/pkg/api" + "github.com/containerd/nri/pkg/log" + "github.com/containerd/nri/pkg/plugin" + yaml "gopkg.in/yaml.v3" +) + +type DefaultValidatorConfig struct { + // Enable the default validator plugin. + Enable bool `yaml:"enable" toml:"enable"` + // RejectOCIHooks fails validation if any plugin injects OCI hooks. + RejectOCIHooks bool `yaml:"rejectOCIHooks" toml:"reject_oci_hooks"` + // RequiredPlugins list globally required plugins. These must be present + // or otherwise validation will fail. + // WARNING: This is a global setting and will affect all containers. In + // particular, if you configure any globally required plugins, you should + // annotate your static pods to tolerate missing plugins. Failing to do + // so will prevent static pods from starting. + // Notes: + // Containers can be annotated to tolerate missing plugins using the + // toleration annotation, if one is set. + RequiredPlugins []string `yaml:"requiredPlugins" toml:"required_plugins"` + // TolerateMissingPlugins is an optional annotation key. If set, it can + // be used to annotate containers to tolerate missing required plugins. + TolerateMissingAnnotation string `yaml:"tolerateMissingPluginsAnnotation" toml:"tolerate_missing_plugins_annotation"` +} + +// DefaultValidator implements default validation. +type DefaultValidator struct { + cfg DefaultValidatorConfig +} + +const ( + // RequiredPlugins is the annotation key for extra required plugins. + RequiredPlugins = plugin.RequiredPluginsAnnotation +) + +var ( + // ErrValidation is returned if validation rejects an adjustment. + ErrValidation = errors.New("validation error") +) + +// NewDefaultValidator creates a new instance of the validator. +func NewDefaultValidator(cfg *DefaultValidatorConfig) *DefaultValidator { + return &DefaultValidator{cfg: *cfg} +} + +// SetConfig sets new configuration for the validator. +func (v *DefaultValidator) SetConfig(cfg *DefaultValidatorConfig) { + if cfg == nil { + return + } + v.cfg = *cfg +} + +// ValidateContainerAdjustment validates a container adjustment. +func (v *DefaultValidator) ValidateContainerAdjustment(ctx context.Context, req *api.ValidateContainerAdjustmentRequest) error { + log.Debugf(ctx, "Validating adjustment of container %s/%s/%s", + req.GetPod().GetNamespace(), req.GetPod().GetName(), req.GetContainer().GetName()) + + if err := v.validateOCIHooks(req); err != nil { + log.Errorf(ctx, "rejecting adjustment: %v", err) + return err + } + + if err := v.validateRequiredPlugins(req); err != nil { + log.Errorf(ctx, "rejecting adjustment: %v", err) + return err + } + + return nil +} + +func (v *DefaultValidator) validateOCIHooks(req *api.ValidateContainerAdjustmentRequest) error { + if req.Adjust == nil { + return nil + } + + if !v.cfg.RejectOCIHooks { + return nil + } + + owners, claimed := req.Owners.HooksOwner(req.Container.Id) + if !claimed { + return nil + } + + offender := "" + + if !strings.Contains(owners, ",") { + offender = fmt.Sprintf("plugin %q", owners) + } else { + offender = fmt.Sprintf("plugins %q", owners) + } + + return fmt.Errorf("%w: %s attempted restricted OCI hook injection", ErrValidation, offender) +} + +func (v *DefaultValidator) validateRequiredPlugins(req *api.ValidateContainerAdjustmentRequest) error { + var ( + container = req.GetContainer().GetName() + required = slices.Clone(v.cfg.RequiredPlugins) + ) + + if tolerateMissing := v.cfg.TolerateMissingAnnotation; tolerateMissing != "" { + value, ok := plugin.GetEffectiveAnnotation(req.GetPod(), tolerateMissing, container) + if ok { + tolerate, err := strconv.ParseBool(value) + if err != nil { + return fmt.Errorf("invalid %s annotation %q: %w", tolerateMissing, value, err) + } + if tolerate { + return nil + } + } + } + + if value, ok := plugin.GetEffectiveAnnotation(req.GetPod(), RequiredPlugins, container); ok { + var annotated []string + if err := yaml.Unmarshal([]byte(value), &annotated); err != nil { + return fmt.Errorf("invalid %s annotation %q: %w", RequiredPlugins, value, err) + } + required = append(required, annotated...) + } + + if len(required) == 0 { + return nil + } + + plugins := req.GetPluginMap() + missing := []string{} + + for _, r := range required { + if _, ok := plugins[r]; !ok { + missing = append(missing, r) + } + } + + if len(missing) == 0 { + return nil + } + + offender := "" + + if len(missing) == 1 { + offender = fmt.Sprintf("required plugin %q", missing[0]) + } else { + offender = fmt.Sprintf("required plugins %q", strings.Join(missing, ",")) + } + + return fmt.Errorf("%w: %s not present", ErrValidation, offender) +} diff --git a/plugins/default-validator/default-validator_test.go b/plugins/default-validator/default-validator_test.go new file mode 100644 index 00000000..315e0777 --- /dev/null +++ b/plugins/default-validator/default-validator_test.go @@ -0,0 +1,256 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package validator + +import ( + "testing" + + "github.com/containerd/nri/pkg/api" + "github.com/stretchr/testify/require" +) + +func TestValidateReqiredPlugins(t *testing.T) { + type testCase struct { + name string + cfg *DefaultValidatorConfig + pod *api.PodSandbox + container *api.Container + plugins []*api.PluginInstance + fail bool + } + + for _, tc := range []*testCase{ + { + name: "no required plugins", + cfg: &DefaultValidatorConfig{ + Enable: true, + }, + pod: &api.PodSandbox{ + Id: "pod-id", + Name: "pod-name", + Namespace: "pod-namespace", + }, + container: &api.Container{ + Id: "container-id", + Name: "container-name", + }, + }, + { + name: "missing annotated required plugin", + cfg: &DefaultValidatorConfig{ + Enable: true, + }, + pod: &api.PodSandbox{ + Id: "pod-id", + Name: "pod-name", + Namespace: "pod-namespace", + Annotations: map[string]string{ + "required-plugins.noderesource.dev/container.container-name": "[ plugin ]", + }, + }, + container: &api.Container{ + Id: "container-id", + Name: "container-name", + }, + fail: true, + }, + { + name: "present annotated required plugin", + cfg: &DefaultValidatorConfig{ + Enable: true, + }, + pod: &api.PodSandbox{ + Id: "pod-id", + Name: "pod-name", + Namespace: "pod-namespace", + Annotations: map[string]string{ + "required-plugins.noderesource.dev/container.container-name": "[ plugin ]", + }, + }, + container: &api.Container{ + Id: "container-id", + Name: "container-name", + }, + plugins: []*api.PluginInstance{ + { + Name: "plugin", + Index: "00", + }, + }, + }, + + { + name: "missing global required plugin", + cfg: &DefaultValidatorConfig{ + Enable: true, + RequiredPlugins: []string{"plugin"}, + }, + pod: &api.PodSandbox{ + Id: "pod-id", + Name: "pod-name", + Namespace: "pod-namespace", + }, + container: &api.Container{ + Id: "container-id", + Name: "container-name", + }, + fail: true, + }, + { + name: "present global required plugin", + cfg: &DefaultValidatorConfig{ + Enable: true, + RequiredPlugins: []string{"plugin"}, + }, + pod: &api.PodSandbox{ + Id: "pod-id", + Name: "pod-name", + Namespace: "pod-namespace", + }, + container: &api.Container{ + Id: "container-id", + Name: "container-name", + }, + plugins: []*api.PluginInstance{ + { + Name: "plugin", + Index: "00", + }, + }, + }, + { + name: "tolerated missing (global required) plugin", + cfg: &DefaultValidatorConfig{ + Enable: true, + RequiredPlugins: []string{"plugin"}, + TolerateMissingAnnotation: "tolerate-missing-plugins", + }, + pod: &api.PodSandbox{ + Id: "pod-id", + Name: "pod-name", + Namespace: "pod-namespace", + Annotations: map[string]string{ + "tolerate-missing-plugins": "true", + }, + }, + container: &api.Container{ + Id: "container-id", + Name: "container-name", + }, + }, + { + name: "present annotated and global required plugin", + cfg: &DefaultValidatorConfig{ + Enable: true, + RequiredPlugins: []string{"plugin1"}, + }, + pod: &api.PodSandbox{ + Id: "pod-id", + Name: "pod-name", + Namespace: "pod-namespace", + Annotations: map[string]string{ + "required-plugins.noderesource.dev/container.container-name": "[ plugin2 ]", + }, + }, + container: &api.Container{ + Id: "container-id", + Name: "container-name", + }, + plugins: []*api.PluginInstance{ + { + Name: "plugin1", + Index: "00", + }, + { + Name: "plugin2", + Index: "01", + }, + }, + }, + { + name: "missing annotated with present global required plugin", + cfg: &DefaultValidatorConfig{ + Enable: true, + RequiredPlugins: []string{"plugin1"}, + }, + pod: &api.PodSandbox{ + Id: "pod-id", + Name: "pod-name", + Namespace: "pod-namespace", + Annotations: map[string]string{ + "required-plugins.noderesource.dev/container.container-name": "[ plugin2 ]", + }, + }, + container: &api.Container{ + Id: "container-id", + Name: "container-name", + }, + plugins: []*api.PluginInstance{ + { + Name: "plugin1", + Index: "00", + }, + }, + fail: true, + }, + { + name: "present annotated with missing global required plugin", + cfg: &DefaultValidatorConfig{ + Enable: true, + RequiredPlugins: []string{"plugin1"}, + }, + pod: &api.PodSandbox{ + Id: "pod-id", + Name: "pod-name", + Namespace: "pod-namespace", + Annotations: map[string]string{ + "required-plugins.noderesource.dev/container.container-name": "[ plugin2 ]", + }, + }, + container: &api.Container{ + Id: "container-id", + Name: "container-name", + }, + plugins: []*api.PluginInstance{ + { + Name: "plugin2", + Index: "00", + }, + }, + fail: true, + }, + } { + t.Run(tc.name, func(t *testing.T) { + var ( + v = NewDefaultValidator(tc.cfg) + req = &api.ValidateContainerAdjustmentRequest{ + Pod: tc.pod, + Container: tc.container, + Plugins: tc.plugins, + } + ) + + err := v.validateRequiredPlugins(req) + if tc.fail { + require.Error(t, err) + } else { + require.NoError(t, err) + } + }) + } + +}