From 859078fbcbf32236afdf53aead3f11d4d0921bc4 Mon Sep 17 00:00:00 2001 From: Krisztian Litkey Date: Thu, 17 Apr 2025 00:21:00 +0300 Subject: [PATCH 1/5] adaptation: implement builtin plugins. Implement a new builtin plugin type. Builtin plugins can be registered from within the runtime during instantiation of the runtime adaptation. We'll use a builtin plugin for default container adjustment validation. Signed-off-by: Krisztian Litkey --- pkg/adaptation/adaptation.go | 24 ++++ pkg/adaptation/api.go | 3 + pkg/adaptation/builtin/plugin.go | 210 +++++++++++++++++++++++++++++++ pkg/adaptation/plugin.go | 40 ++++-- pkg/adaptation/plugin_type.go | 95 +++++++++++--- 5 files changed, 345 insertions(+), 27 deletions(-) create mode 100644 pkg/adaptation/builtin/plugin.go diff --git a/pkg/adaptation/adaptation.go b/pkg/adaptation/adaptation.go index e46c351f..8195e101 100644 --- a/pkg/adaptation/adaptation.go +++ b/pkg/adaptation/adaptation.go @@ -27,6 +27,7 @@ import ( "sort" "sync" + "github.com/containerd/nri/pkg/adaptation/builtin" "github.com/containerd/nri/pkg/api" "github.com/containerd/nri/pkg/log" "github.com/containerd/ttrpc" @@ -70,6 +71,7 @@ type Adaptation struct { listener net.Listener plugins []*plugin validators []*plugin + builtin []*builtin.BuiltinPlugin syncLock sync.RWMutex wasmService *api.PluginPlugin } @@ -123,6 +125,14 @@ func WithTTRPCOptions(clientOpts []ttrpc.ClientOpts, serverOpts []ttrpc.ServerOp } } +// WithBuiltinPlugins sets extra builtin plugins to register. +func WithBuiltinPlugins(plugins ...*builtin.BuiltinPlugin) Option { + return func(r *Adaptation) error { + r.builtin = append(r.builtin, plugins...) + return nil + } +} + // New creates a new NRI Runtime. func New(name, version string, syncFn SyncFn, updateFn UpdateFn, opts ...Option) (*Adaptation, error) { var err error @@ -433,6 +443,20 @@ func (r *Adaptation) startPlugins() (retErr error) { } }() + for _, b := range r.builtin { + log.Infof(noCtx, "starting builtin NRI plugin %q...", b.Index+"-"+b.Base) + p, err := r.newBuiltinPlugin(b) + if err != nil { + return fmt.Errorf("failed to initialize builtin NRI plugin %q: %v", b.Base, err) + } + + if err := p.start(r.name, r.version); err != nil { + return fmt.Errorf("failed to start builtin NRI plugin %q: %v", b.Base, err) + } + + plugins = append(plugins, p) + } + for i, name := range names { log.Infof(noCtx, "starting pre-installed NRI plugin %q...", name) diff --git a/pkg/adaptation/api.go b/pkg/adaptation/api.go index eeab0887..e4a432c0 100644 --- a/pkg/adaptation/api.go +++ b/pkg/adaptation/api.go @@ -37,6 +37,9 @@ type ( SynchronizeRequest = api.SynchronizeRequest SynchronizeResponse = api.SynchronizeResponse + ShutdownRequest = api.Empty + ShutdownResponse = api.Empty + CreateContainerRequest = api.CreateContainerRequest CreateContainerResponse = api.CreateContainerResponse UpdateContainerRequest = api.UpdateContainerRequest diff --git a/pkg/adaptation/builtin/plugin.go b/pkg/adaptation/builtin/plugin.go new file mode 100644 index 00000000..d818c182 --- /dev/null +++ b/pkg/adaptation/builtin/plugin.go @@ -0,0 +1,210 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package builtin + +import ( + "context" + + "github.com/containerd/nri/pkg/api" +) + +type BuiltinPlugin struct { + Base string + Index string + Handlers BuiltinHandlers +} + +type BuiltinHandlers struct { + Configure func(context.Context, *api.ConfigureRequest) (*api.ConfigureResponse, error) + Synchronize func(context.Context, *api.SynchronizeRequest) (*api.SynchronizeResponse, error) + RunPodSandbox func(context.Context, *api.RunPodSandboxRequest) error + StopPodSandbox func(context.Context, *api.StopPodSandboxRequest) error + RemovePodSandbox func(context.Context, *api.RemovePodSandboxRequest) error + UpdatePodSandbox func(context.Context, *api.UpdatePodSandboxRequest) (*api.UpdatePodSandboxResponse, error) + PostUpdatePodSandbox func(context.Context, *api.PostUpdatePodSandboxRequest) error + + CreateContainer func(context.Context, *api.CreateContainerRequest) (*api.CreateContainerResponse, error) + PostCreateContainer func(context.Context, *api.PostCreateContainerRequest) error + StartContainer func(context.Context, *api.StartContainerRequest) error + PostStartContainer func(context.Context, *api.PostStartContainerRequest) error + UpdateContainer func(context.Context, *api.UpdateContainerRequest) (*api.UpdateContainerResponse, error) + PostUpdateContainer func(context.Context, *api.PostUpdateContainerRequest) error + StopContainer func(context.Context, *api.StopContainerRequest) (*api.StopContainerResponse, error) + RemoveContainer func(context.Context, *api.RemoveContainerRequest) error + ValidateContainerAdjustment func(context.Context, *api.ValidateContainerAdjustmentRequest) error +} + +func (b *BuiltinPlugin) Configure(ctx context.Context, req *api.ConfigureRequest) (*api.ConfigureResponse, error) { + var ( + rpl = &api.ConfigureResponse{} + err error + ) + + if b.Handlers.Configure != nil { + rpl, err = b.Handlers.Configure(ctx, req) + } + + if rpl.Events == 0 { + var events api.EventMask + + if b.Handlers.RunPodSandbox != nil { + events.Set(api.Event_RUN_POD_SANDBOX) + } + if b.Handlers.StopPodSandbox != nil { + events.Set(api.Event_STOP_POD_SANDBOX) + } + if b.Handlers.RemovePodSandbox != nil { + events.Set(api.Event_REMOVE_POD_SANDBOX) + } + if b.Handlers.UpdatePodSandbox != nil { + events.Set(api.Event_UPDATE_POD_SANDBOX) + } + if b.Handlers.PostUpdatePodSandbox != nil { + events.Set(api.Event_POST_UPDATE_POD_SANDBOX) + } + if b.Handlers.CreateContainer != nil { + events.Set(api.Event_CREATE_CONTAINER) + } + if b.Handlers.PostCreateContainer != nil { + events.Set(api.Event_POST_CREATE_CONTAINER) + } + if b.Handlers.StartContainer != nil { + events.Set(api.Event_START_CONTAINER) + } + if b.Handlers.PostStartContainer != nil { + events.Set(api.Event_POST_START_CONTAINER) + } + if b.Handlers.UpdateContainer != nil { + events.Set(api.Event_UPDATE_CONTAINER) + } + if b.Handlers.PostUpdateContainer != nil { + events.Set(api.Event_POST_UPDATE_CONTAINER) + } + if b.Handlers.StopContainer != nil { + events.Set(api.Event_STOP_CONTAINER) + } + if b.Handlers.RemoveContainer != nil { + events.Set(api.Event_REMOVE_CONTAINER) + } + if b.Handlers.ValidateContainerAdjustment != nil { + events.Set(api.Event_VALIDATE_CONTAINER_ADJUSTMENT) + } + + rpl.Events = int32(events) + } + + return rpl, err +} + +func (b *BuiltinPlugin) Synchronize(ctx context.Context, req *api.SynchronizeRequest) (*api.SynchronizeResponse, error) { + if b.Handlers.Synchronize != nil { + return b.Handlers.Synchronize(ctx, req) + } + return &api.SynchronizeResponse{}, nil +} + +func (b *BuiltinPlugin) Shutdown(context.Context, *api.ShutdownRequest) (*api.ShutdownResponse, error) { + return &api.ShutdownResponse{}, nil +} + +func (b *BuiltinPlugin) CreateContainer(ctx context.Context, req *api.CreateContainerRequest) (*api.CreateContainerResponse, error) { + if b.Handlers.CreateContainer != nil { + return b.Handlers.CreateContainer(ctx, req) + } + return &api.CreateContainerResponse{}, nil +} + +func (b *BuiltinPlugin) UpdateContainer(ctx context.Context, req *api.UpdateContainerRequest) (*api.UpdateContainerResponse, error) { + if b.Handlers.UpdateContainer != nil { + return b.Handlers.UpdateContainer(ctx, req) + } + return &api.UpdateContainerResponse{}, nil +} + +func (b *BuiltinPlugin) StopContainer(ctx context.Context, req *api.StopContainerRequest) (*api.StopContainerResponse, error) { + if b.Handlers.StopContainer != nil { + return b.Handlers.StopContainer(ctx, req) + } + return &api.StopContainerResponse{}, nil +} + +func (b *BuiltinPlugin) StateChange(ctx context.Context, evt *api.StateChangeEvent) (*api.StateChangeResponse, error) { + var err error + switch evt.Event { + case api.Event_RUN_POD_SANDBOX: + if b.Handlers.RunPodSandbox != nil { + err = b.Handlers.RunPodSandbox(ctx, evt) + } + case api.Event_STOP_POD_SANDBOX: + if b.Handlers.StopPodSandbox != nil { + err = b.Handlers.StopPodSandbox(ctx, evt) + } + case api.Event_REMOVE_POD_SANDBOX: + if b.Handlers.RemovePodSandbox != nil { + err = b.Handlers.RemovePodSandbox(ctx, evt) + } + case api.Event_POST_CREATE_CONTAINER: + if b.Handlers.PostCreateContainer != nil { + err = b.Handlers.PostCreateContainer(ctx, evt) + } + case api.Event_START_CONTAINER: + if b.Handlers.StartContainer != nil { + err = b.Handlers.StartContainer(ctx, evt) + } + case api.Event_POST_START_CONTAINER: + if b.Handlers.PostStartContainer != nil { + err = b.Handlers.PostStartContainer(ctx, evt) + } + case api.Event_POST_UPDATE_CONTAINER: + if b.Handlers.PostUpdateContainer != nil { + err = b.Handlers.PostUpdateContainer(ctx, evt) + } + case api.Event_REMOVE_CONTAINER: + if b.Handlers.RemoveContainer != nil { + err = b.Handlers.RemoveContainer(ctx, evt) + } + } + + return &api.StateChangeResponse{}, err +} + +func (b *BuiltinPlugin) UpdatePodSandbox(ctx context.Context, req *api.UpdatePodSandboxRequest) (*api.UpdatePodSandboxResponse, error) { + if b.Handlers.UpdatePodSandbox != nil { + return b.Handlers.UpdatePodSandbox(ctx, req) + } + return &api.UpdatePodSandboxResponse{}, nil +} + +func (b *BuiltinPlugin) PostUpdatePodSandbox(ctx context.Context, req *api.PostUpdatePodSandboxRequest) error { + if b.Handlers.PostUpdatePodSandbox != nil { + return b.Handlers.PostUpdatePodSandbox(ctx, req) + } + return nil +} + +func (b *BuiltinPlugin) ValidateContainerAdjustment(ctx context.Context, req *api.ValidateContainerAdjustmentRequest) (*api.ValidateContainerAdjustmentResponse, error) { + if b.Handlers.ValidateContainerAdjustment != nil { + if err := b.Handlers.ValidateContainerAdjustment(ctx, req); err != nil { + return &api.ValidateContainerAdjustmentResponse{ + Reject: true, + Reason: err.Error(), + }, nil + } + } + + return &api.ValidateContainerAdjustmentResponse{}, nil +} diff --git a/pkg/adaptation/plugin.go b/pkg/adaptation/plugin.go index 7954b864..80e5555b 100644 --- a/pkg/adaptation/plugin.go +++ b/pkg/adaptation/plugin.go @@ -28,6 +28,7 @@ import ( "sync" "time" + "github.com/containerd/nri/pkg/adaptation/builtin" "github.com/containerd/nri/pkg/api" "github.com/containerd/nri/pkg/log" "github.com/containerd/nri/pkg/net" @@ -165,6 +166,20 @@ func (r *Adaptation) newLaunchedPlugin(dir, idx, base, cfg string) (p *plugin, r return p, nil } +func (r *Adaptation) newBuiltinPlugin(b *builtin.BuiltinPlugin) (*plugin, error) { + if b.Base == "" || b.Index == "" { + return nil, fmt.Errorf("builtin plugin without index or name (%q, %q)", b.Index, b.Base) + } + + return &plugin{ + idx: b.Index, + base: b.Base, + closeC: make(chan struct{}), + r: r, + impl: &pluginType{builtinImpl: b}, + }, nil +} + func isWasm(path string) bool { file, err := os.Open(path) if err != nil { @@ -296,7 +311,7 @@ func (p *plugin) connect(conn stdnet.Conn) (retErr error) { // Start Runtime service, wait for plugin to register, then configure it. func (p *plugin) start(name, version string) (err error) { - // skip start for WASM plugins and head right to the registration for + // skip start for WASM and builtin plugins and head right to the registration for // events config if p.impl.isTtrpc() { var ( @@ -340,7 +355,7 @@ func (p *plugin) start(name, version string) (err error) { // close a plugin shutting down its multiplexed ttrpc connections. func (p *plugin) close() { - if p.impl.isWasm() { + if p.impl.isWasm() || p.impl.isBuiltin() { p.closed = true return } @@ -366,7 +381,7 @@ func (p *plugin) isClosed() bool { // stop a plugin (if it was launched by us) func (p *plugin) stop() error { - if p.isExternal() || p.cmd.Process == nil || p.impl.isWasm() { + if p.isExternal() || p.cmd.Process == nil || p.impl.isWasm() || p.impl.isBuiltin() { return nil } @@ -389,11 +404,20 @@ func (p *plugin) name() string { } func (p *plugin) qualifiedName() string { - var kind, idx, base string - if p.isExternal() { - kind = "external" + var kind, idx, base, pid string + if p.impl.isBuiltin() { + kind = "builtin" } else { - kind = "pre-connected" + if p.isExternal() { + kind = "external" + } else { + kind = "pre-connected" + } + if p.impl.isWasm() { + kind += "-wasm" + } else { + pid = "[" + strconv.Itoa(p.pid) + "]" + } } if idx = p.idx; idx == "" { idx = "??" @@ -401,7 +425,7 @@ func (p *plugin) qualifiedName() string { if base = p.base; base == "" { base = "plugin" } - return kind + ":" + idx + "-" + base + "[" + strconv.Itoa(p.pid) + "]" + return kind + ":" + idx + "-" + base + pid } // RegisterPlugin handles the plugin's registration request. diff --git a/pkg/adaptation/plugin_type.go b/pkg/adaptation/plugin_type.go index cfc4c44c..43390291 100644 --- a/pkg/adaptation/plugin_type.go +++ b/pkg/adaptation/plugin_type.go @@ -18,15 +18,21 @@ package adaptation import ( "context" + "errors" "github.com/containerd/nri/pkg/api" ) type pluginType struct { - wasmImpl api.Plugin - ttrpcImpl api.PluginService + wasmImpl api.Plugin + ttrpcImpl api.PluginService + builtinImpl api.PluginService } +var ( + errUnknownImpl = errors.New("unknown plugin implementation type") +) + func (p *pluginType) isWasm() bool { return p.wasmImpl != nil } @@ -35,60 +41,111 @@ func (p *pluginType) isTtrpc() bool { return p.ttrpcImpl != nil } +func (p *pluginType) isBuiltin() bool { + return p.builtinImpl != nil +} + func (p *pluginType) Synchronize(ctx context.Context, req *SynchronizeRequest) (*SynchronizeResponse, error) { - if p.wasmImpl != nil { + switch { + case p.ttrpcImpl != nil: + return p.ttrpcImpl.Synchronize(ctx, req) + case p.builtinImpl != nil: + return p.builtinImpl.Synchronize(ctx, req) + case p.wasmImpl != nil: return p.wasmImpl.Synchronize(ctx, req) } - return p.ttrpcImpl.Synchronize(ctx, req) + + return nil, errUnknownImpl } func (p *pluginType) Configure(ctx context.Context, req *ConfigureRequest) (*ConfigureResponse, error) { - if p.wasmImpl != nil { + switch { + case p.ttrpcImpl != nil: + return p.ttrpcImpl.Configure(ctx, req) + case p.builtinImpl != nil: + return p.builtinImpl.Configure(ctx, req) + case p.wasmImpl != nil: return p.wasmImpl.Configure(ctx, req) } - return p.ttrpcImpl.Configure(ctx, req) + + return nil, errUnknownImpl } func (p *pluginType) CreateContainer(ctx context.Context, req *CreateContainerRequest) (*CreateContainerResponse, error) { - if p.wasmImpl != nil { + switch { + case p.ttrpcImpl != nil: + return p.ttrpcImpl.CreateContainer(ctx, req) + case p.builtinImpl != nil: + return p.builtinImpl.CreateContainer(ctx, req) + case p.wasmImpl != nil: return p.wasmImpl.CreateContainer(ctx, req) } - return p.ttrpcImpl.CreateContainer(ctx, req) + + return nil, errUnknownImpl } func (p *pluginType) UpdateContainer(ctx context.Context, req *UpdateContainerRequest) (*UpdateContainerResponse, error) { - if p.wasmImpl != nil { + switch { + case p.ttrpcImpl != nil: + return p.ttrpcImpl.UpdateContainer(ctx, req) + case p.builtinImpl != nil: + return p.builtinImpl.UpdateContainer(ctx, req) + case p.wasmImpl != nil: return p.wasmImpl.UpdateContainer(ctx, req) } - return p.ttrpcImpl.UpdateContainer(ctx, req) + + return nil, errUnknownImpl } func (p *pluginType) StopContainer(ctx context.Context, req *StopContainerRequest) (*StopContainerResponse, error) { - if p.wasmImpl != nil { + switch { + case p.ttrpcImpl != nil: + return p.ttrpcImpl.StopContainer(ctx, req) + case p.builtinImpl != nil: + return p.builtinImpl.StopContainer(ctx, req) + case p.wasmImpl != nil: return p.wasmImpl.StopContainer(ctx, req) } - return p.ttrpcImpl.StopContainer(ctx, req) + + return nil, errUnknownImpl } func (p *pluginType) UpdatePodSandbox(ctx context.Context, req *UpdatePodSandboxRequest) (*UpdatePodSandboxResponse, error) { - if p.wasmImpl != nil { + switch { + case p.ttrpcImpl != nil: + return p.ttrpcImpl.UpdatePodSandbox(ctx, req) + case p.builtinImpl != nil: + return p.builtinImpl.UpdatePodSandbox(ctx, req) + case p.wasmImpl != nil: return p.wasmImpl.UpdatePodSandbox(ctx, req) } - return p.ttrpcImpl.UpdatePodSandbox(ctx, req) + + return nil, errUnknownImpl } func (p *pluginType) StateChange(ctx context.Context, req *StateChangeEvent) (err error) { - if p.wasmImpl != nil { - _, err = p.wasmImpl.StateChange(ctx, req) - } else { + switch { + case p.ttrpcImpl != nil: _, err = p.ttrpcImpl.StateChange(ctx, req) + case p.builtinImpl != nil: + _, err = p.builtinImpl.StateChange(ctx, req) + case p.wasmImpl != nil: + _, err = p.wasmImpl.StateChange(ctx, req) + default: + err = errUnknownImpl } return err } func (p *pluginType) ValidateContainerAdjustment(ctx context.Context, req *ValidateContainerAdjustmentRequest) (*ValidateContainerAdjustmentResponse, error) { - if p.wasmImpl != nil { + switch { + case p.ttrpcImpl != nil: + return p.ttrpcImpl.ValidateContainerAdjustment(ctx, req) + case p.builtinImpl != nil: + return p.builtinImpl.ValidateContainerAdjustment(ctx, req) + case p.wasmImpl != nil: return p.wasmImpl.ValidateContainerAdjustment(ctx, req) } - return p.ttrpcImpl.ValidateContainerAdjustment(ctx, req) + + return nil, errUnknownImpl } From c485d5fbddea569dfb1feec9b9e42e8d41b5b3af Mon Sep 17 00:00:00 2001 From: Krisztian Litkey Date: Thu, 17 Apr 2025 02:57:11 +0300 Subject: [PATCH 2/5] adaptation, plugins: implement default validation. Implement default (container creation/adjustment) validation as a builtin plugin. The default validator can be configured to reject OCI hook injection. Additionally, containers can be annotated with a set of required plugins. If annotated, these plugins must be present during container creation or else the creation of the container is rejected by the validator. Signed-off-by: Krisztian Litkey --- go.mod | 2 +- pkg/adaptation/adaptation.go | 11 ++ pkg/api/validate.go | 14 ++ pkg/plugin/annotations.go | 59 ++++++ plugins/default-validator/builtin/plugin.go | 48 +++++ .../default-validator/default-validator.go | 177 ++++++++++++++++++ 6 files changed, 310 insertions(+), 1 deletion(-) create mode 100644 pkg/plugin/annotations.go create mode 100644 plugins/default-validator/builtin/plugin.go create mode 100644 plugins/default-validator/default-validator.go diff --git a/go.mod b/go.mod index 369346c6..5f29dfb9 100644 --- a/go.mod +++ b/go.mod @@ -17,6 +17,7 @@ require ( golang.org/x/sys v0.21.0 google.golang.org/grpc v1.57.1 google.golang.org/protobuf v1.34.1 + gopkg.in/yaml.v3 v3.0.1 ) require ( @@ -35,7 +36,6 @@ require ( golang.org/x/tools v0.21.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20230731190214-cbb8c96f2d6d // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect ) replace github.com/opencontainers/runtime-tools v0.9.0 => github.com/opencontainers/runtime-tools v0.0.0-20221026201742-946c877fa809 diff --git a/pkg/adaptation/adaptation.go b/pkg/adaptation/adaptation.go index 8195e101..7ae20c12 100644 --- a/pkg/adaptation/adaptation.go +++ b/pkg/adaptation/adaptation.go @@ -30,6 +30,7 @@ import ( "github.com/containerd/nri/pkg/adaptation/builtin" "github.com/containerd/nri/pkg/api" "github.com/containerd/nri/pkg/log" + validator "github.com/containerd/nri/plugins/default-validator/builtin" "github.com/containerd/ttrpc" "github.com/tetratelabs/wazero" "github.com/tetratelabs/wazero/imports/wasi_snapshot_preview1" @@ -133,6 +134,16 @@ func WithBuiltinPlugins(plugins ...*builtin.BuiltinPlugin) Option { } } +// WithDefaultValidator sets up builtin validator plugin if it is configured. +func WithDefaultValidator(cfg *validator.DefaultValidatorConfig) Option { + return func(r *Adaptation) error { + if plugin := validator.GetDefaultValidator(cfg); plugin != nil { + r.builtin = append([]*builtin.BuiltinPlugin{plugin}, r.builtin...) + } + return nil + } +} + // New creates a new NRI Runtime. func New(name, version string, syncFn SyncFn, updateFn UpdateFn, opts ...Option) (*Adaptation, error) { var err error diff --git a/pkg/api/validate.go b/pkg/api/validate.go index 1d636dda..aa7d68c8 100644 --- a/pkg/api/validate.go +++ b/pkg/api/validate.go @@ -48,3 +48,17 @@ func (v *ValidateContainerAdjustmentResponse) ValidationResult(plugin string) er return fmt.Errorf("validator %q rejected container adjustment, reason: %s", plugin, reason) } + +func (v *ValidateContainerAdjustmentRequest) GetPluginMap() map[string]*PluginInstance { + if v == nil { + return nil + } + + plugins := make(map[string]*PluginInstance) + for _, p := range v.Plugins { + plugins[p.Name] = &PluginInstance{Name: p.Name} + plugins[p.Index+"-"+p.Name] = p + } + + return plugins +} diff --git a/pkg/plugin/annotations.go b/pkg/plugin/annotations.go new file mode 100644 index 00000000..45160941 --- /dev/null +++ b/pkg/plugin/annotations.go @@ -0,0 +1,59 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package plugin + +import ( + "github.com/containerd/nri/pkg/api" +) + +const ( + // AnnotationDomain is the domain used for NRI-specific annotations. + AnnotationDomain = "noderesource.dev" + + // RequiredPluginsAnnotation can be used to annotate pods with a list + // of pod- or container-specific plugins which must process containers + // during creation. If enabled, the default validator checks for this + // and rejects the creation of containers which fail this check. + RequiredPluginsAnnotation = "required-plugins." + AnnotationDomain +) + +// GetEffectiveAnnotation retrieves a custom annotation from a pod which +// applies to given container. The syntax allows both pod- and container- +// scoped annotations. Container-scoped annotations take precedence over +// pod-scoped ones. The key syntax defines the scope of the annotation. +// - container-scope: /container. +// - pod-scope: /pod, or just +func GetEffectiveAnnotation(pod *api.PodSandbox, key, container string) (string, bool) { + annotations := pod.GetAnnotations() + if len(annotations) == 0 { + return "", false + } + + keys := []string{ + key + "/container." + container, + key + "/pod", + key, + } + + for _, k := range keys { + if v, ok := annotations[k]; ok { + return v, true + } + } + + return "", false +} diff --git a/plugins/default-validator/builtin/plugin.go b/plugins/default-validator/builtin/plugin.go new file mode 100644 index 00000000..d391bbcc --- /dev/null +++ b/plugins/default-validator/builtin/plugin.go @@ -0,0 +1,48 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package builtin + +import ( + "context" + + "github.com/containerd/nri/pkg/adaptation/builtin" + "github.com/containerd/nri/pkg/log" + + validator "github.com/containerd/nri/plugins/default-validator" +) + +type ( + DefaultValidatorConfig = validator.DefaultValidatorConfig +) + +// GetDefaultValidator returns a configured instance of the default validator. +// If default validation is disabled nil is returned. +func GetDefaultValidator(cfg *DefaultValidatorConfig) *builtin.BuiltinPlugin { + if cfg == nil || !cfg.Enable { + log.Infof(context.TODO(), "built-in NRI default validator is disabled") + return nil + } + + v := validator.NewDefaultValidator(cfg) + return &builtin.BuiltinPlugin{ + Base: "default-validator", + Index: "00", + Handlers: builtin.BuiltinHandlers{ + ValidateContainerAdjustment: v.ValidateContainerAdjustment, + }, + } +} diff --git a/plugins/default-validator/default-validator.go b/plugins/default-validator/default-validator.go new file mode 100644 index 00000000..cc9d404a --- /dev/null +++ b/plugins/default-validator/default-validator.go @@ -0,0 +1,177 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package validator + +import ( + "context" + "errors" + "fmt" + "slices" + "strconv" + "strings" + + "github.com/containerd/nri/pkg/api" + "github.com/containerd/nri/pkg/log" + "github.com/containerd/nri/pkg/plugin" + yaml "gopkg.in/yaml.v3" +) + +type DefaultValidatorConfig struct { + // Enable the default validator plugin. + Enable bool `yaml:"enable" toml:"enable"` + // RejectOCIHooks fails validation if any plugin injects OCI hooks. + RejectOCIHooks bool `yaml:"rejectOCIHooks" toml:"reject_oci_hooks"` + // RequiredPlugins list globally required plugins. These must be present + // or otherwise validation will fail. + // WARNING: This is a global setting and will affect all containers. In + // particular, if you configure any globally required plugins, you should + // annotate your static pods to tolerate missing plugins. Failing to do + // so will prevent static pods from starting. + // Notes: + // Containers can be annotated to tolerate missing plugins using the + // toleration annotation, if one is set. + RequiredPlugins []string `yaml:"requiredPlugins" toml:"required_plugins"` + // TolerateMissingPlugins is an optional annotation key. If set, it can + // be used to annotate containers to tolerate missing required plugins. + TolerateMissingAnnotation string `yaml:"tolerateMissingPluginsAnnotation" toml:"tolerate_missing_plugins_annotation"` +} + +// DefaultValidator implements default validation. +type DefaultValidator struct { + cfg DefaultValidatorConfig +} + +const ( + // RequiredPlugins is the annotation key for extra required plugins. + RequiredPlugins = plugin.RequiredPluginsAnnotation +) + +var ( + // ErrValidation is returned if validation rejects an adjustment. + ErrValidation = errors.New("validation error") +) + +// NewDefaultValidator creates a new instance of the validator. +func NewDefaultValidator(cfg *DefaultValidatorConfig) *DefaultValidator { + return &DefaultValidator{cfg: *cfg} +} + +// SetConfig sets new configuration for the validator. +func (v *DefaultValidator) SetConfig(cfg *DefaultValidatorConfig) { + if cfg == nil { + return + } + v.cfg = *cfg +} + +// ValidateContainerAdjustment validates a container adjustment. +func (v *DefaultValidator) ValidateContainerAdjustment(ctx context.Context, req *api.ValidateContainerAdjustmentRequest) error { + log.Debugf(ctx, "Validating adjustment of container %s/%s/%s", + req.GetPod().GetNamespace(), req.GetPod().GetName(), req.GetContainer().GetName()) + + if err := v.validateOCIHooks(req); err != nil { + log.Errorf(ctx, "rejecting adjustment: %v", err) + return err + } + + if err := v.validateRequiredPlugins(req); err != nil { + log.Errorf(ctx, "rejecting adjustment: %v", err) + return err + } + + return nil +} + +func (v *DefaultValidator) validateOCIHooks(req *api.ValidateContainerAdjustmentRequest) error { + if req.Adjust == nil { + return nil + } + + if !v.cfg.RejectOCIHooks { + return nil + } + + owners, claimed := req.Owners.HooksOwner(req.Container.Id) + if !claimed { + return nil + } + + offender := "" + + if !strings.Contains(owners, ",") { + offender = fmt.Sprintf("plugin %q", owners) + } else { + offender = fmt.Sprintf("plugins %q", owners) + } + + return fmt.Errorf("%w: %s attempted restricted OCI hook injection", ErrValidation, offender) +} + +func (v *DefaultValidator) validateRequiredPlugins(req *api.ValidateContainerAdjustmentRequest) error { + var ( + container = req.GetContainer().GetName() + required = slices.Clone(v.cfg.RequiredPlugins) + ) + + if tolerateMissing := v.cfg.TolerateMissingAnnotation; tolerateMissing != "" { + value, ok := plugin.GetEffectiveAnnotation(req.GetPod(), tolerateMissing, container) + if ok { + tolerate, err := strconv.ParseBool(value) + if err != nil { + return fmt.Errorf("invalid %s annotation %q: %w", tolerateMissing, value, err) + } + if tolerate { + return nil + } + } + } + + if value, ok := plugin.GetEffectiveAnnotation(req.GetPod(), RequiredPlugins, container); ok { + var annotated []string + if err := yaml.Unmarshal([]byte(value), &annotated); err != nil { + return fmt.Errorf("invalid %s annotation %q: %w", RequiredPlugins, value, err) + } + required = append(required, annotated...) + } + + if len(required) == 0 { + return nil + } + + plugins := req.GetPluginMap() + missing := []string{} + + for _, r := range required { + if _, ok := plugins[r]; !ok { + missing = append(missing, r) + } + } + + if len(missing) == 0 { + return nil + } + + offender := "" + + if len(missing) == 1 { + offender = fmt.Sprintf("required plugin %q", missing[0]) + } else { + offender = fmt.Sprintf("required plugins %q", strings.Join(missing, ",")) + } + + return fmt.Errorf("%w: %s not present", ErrValidation, offender) +} From 4c20b621dd9253f4e4903092a5f77e2f0822a04f Mon Sep 17 00:00:00 2001 From: Krisztian Litkey Date: Sat, 17 May 2025 22:08:32 +0300 Subject: [PATCH 3/5] api: make field owner lookup purely read-only. Split per container FieldOwners lookup for plugin field claiming and clearing from that of plugin field owner lookup so that the formers create missing owners but the latter does not. This makes field owner lookup a purely read-only operation, avoiding a data read/write race when we have both builtin and external validators present and we run validation in parallel on multiple goroutines. Also, do not export the new ownersFor and mustOwnersFor functions as they are never used externally. Signed-off-by: Krisztian Litkey --- pkg/api/owners.go | 138 ++++++++++++++++++++++++---------------------- 1 file changed, 73 insertions(+), 65 deletions(-) diff --git a/pkg/api/owners.go b/pkg/api/owners.go index e476ac3d..af74c216 100644 --- a/pkg/api/owners.go +++ b/pkg/api/owners.go @@ -38,262 +38,262 @@ func NewOwningPlugins() *OwningPlugins { } func (o *OwningPlugins) ClaimAnnotation(id, key, plugin string) error { - return o.OwnersFor(id).ClaimAnnotation(key, plugin) + return o.mustOwnersFor(id).ClaimAnnotation(key, plugin) } func (o *OwningPlugins) ClaimMount(id, destination, plugin string) error { - return o.OwnersFor(id).ClaimMount(destination, plugin) + return o.mustOwnersFor(id).ClaimMount(destination, plugin) } func (o *OwningPlugins) ClaimHooks(id, plugin string) error { - return o.OwnersFor(id).ClaimHooks(plugin) + return o.mustOwnersFor(id).ClaimHooks(plugin) } func (o *OwningPlugins) ClaimDevice(id, path, plugin string) error { - return o.OwnersFor(id).ClaimDevice(path, plugin) + return o.mustOwnersFor(id).ClaimDevice(path, plugin) } func (o *OwningPlugins) ClaimCdiDevice(id, name, plugin string) error { - return o.OwnersFor(id).ClaimCdiDevice(name, plugin) + return o.mustOwnersFor(id).ClaimCdiDevice(name, plugin) } func (o *OwningPlugins) ClaimEnv(id, name, plugin string) error { - return o.OwnersFor(id).ClaimEnv(name, plugin) + return o.mustOwnersFor(id).ClaimEnv(name, plugin) } func (o *OwningPlugins) ClaimArgs(id, plugin string) error { - return o.OwnersFor(id).ClaimArgs(plugin) + return o.mustOwnersFor(id).ClaimArgs(plugin) } func (o *OwningPlugins) ClaimMemLimit(id, plugin string) error { - return o.OwnersFor(id).ClaimMemLimit(plugin) + return o.mustOwnersFor(id).ClaimMemLimit(plugin) } func (o *OwningPlugins) ClaimMemReservation(id, plugin string) error { - return o.OwnersFor(id).ClaimMemReservation(plugin) + return o.mustOwnersFor(id).ClaimMemReservation(plugin) } func (o *OwningPlugins) ClaimMemSwapLimit(id, plugin string) error { - return o.OwnersFor(id).ClaimMemSwapLimit(plugin) + return o.mustOwnersFor(id).ClaimMemSwapLimit(plugin) } func (o *OwningPlugins) ClaimMemKernelLimit(id, plugin string) error { - return o.OwnersFor(id).ClaimMemKernelLimit(plugin) + return o.mustOwnersFor(id).ClaimMemKernelLimit(plugin) } func (o *OwningPlugins) ClaimMemTCPLimit(id, plugin string) error { - return o.OwnersFor(id).ClaimMemTCPLimit(plugin) + return o.mustOwnersFor(id).ClaimMemTCPLimit(plugin) } func (o *OwningPlugins) ClaimMemSwappiness(id, plugin string) error { - return o.OwnersFor(id).ClaimMemSwappiness(plugin) + return o.mustOwnersFor(id).ClaimMemSwappiness(plugin) } func (o *OwningPlugins) ClaimMemDisableOomKiller(id, plugin string) error { - return o.OwnersFor(id).ClaimMemDisableOomKiller(plugin) + return o.mustOwnersFor(id).ClaimMemDisableOomKiller(plugin) } func (o *OwningPlugins) ClaimMemUseHierarchy(id, plugin string) error { - return o.OwnersFor(id).ClaimMemUseHierarchy(plugin) + return o.mustOwnersFor(id).ClaimMemUseHierarchy(plugin) } func (o *OwningPlugins) ClaimCPUShares(id, plugin string) error { - return o.OwnersFor(id).ClaimCPUShares(plugin) + return o.mustOwnersFor(id).ClaimCPUShares(plugin) } func (o *OwningPlugins) ClaimCPUQuota(id, plugin string) error { - return o.OwnersFor(id).ClaimCPUQuota(plugin) + return o.mustOwnersFor(id).ClaimCPUQuota(plugin) } func (o *OwningPlugins) ClaimCPUPeriod(id, plugin string) error { - return o.OwnersFor(id).ClaimCPUPeriod(plugin) + return o.mustOwnersFor(id).ClaimCPUPeriod(plugin) } func (o *OwningPlugins) ClaimCPURealtimeRuntime(id, plugin string) error { - return o.OwnersFor(id).ClaimCPURealtimeRuntime(plugin) + return o.mustOwnersFor(id).ClaimCPURealtimeRuntime(plugin) } func (o *OwningPlugins) ClaimCPURealtimePeriod(id, plugin string) error { - return o.OwnersFor(id).ClaimCPURealtimePeriod(plugin) + return o.mustOwnersFor(id).ClaimCPURealtimePeriod(plugin) } func (o *OwningPlugins) ClaimCPUSetCPUs(id, plugin string) error { - return o.OwnersFor(id).ClaimCPUSetCPUs(plugin) + return o.mustOwnersFor(id).ClaimCPUSetCPUs(plugin) } func (o *OwningPlugins) ClaimCPUSetMems(id, plugin string) error { - return o.OwnersFor(id).ClaimCPUSetMems(plugin) + return o.mustOwnersFor(id).ClaimCPUSetMems(plugin) } func (o *OwningPlugins) ClaimPidsLimit(id, plugin string) error { - return o.OwnersFor(id).ClaimPidsLimit(plugin) + return o.mustOwnersFor(id).ClaimPidsLimit(plugin) } func (o *OwningPlugins) ClaimHugepageLimit(id, size, plugin string) error { - return o.OwnersFor(id).ClaimHugepageLimit(size, plugin) + return o.mustOwnersFor(id).ClaimHugepageLimit(size, plugin) } func (o *OwningPlugins) ClaimBlockioClass(id, plugin string) error { - return o.OwnersFor(id).ClaimBlockioClass(plugin) + return o.mustOwnersFor(id).ClaimBlockioClass(plugin) } func (o *OwningPlugins) ClaimRdtClass(id, plugin string) error { - return o.OwnersFor(id).ClaimRdtClass(plugin) + return o.mustOwnersFor(id).ClaimRdtClass(plugin) } func (o *OwningPlugins) ClaimCgroupsUnified(id, key, plugin string) error { - return o.OwnersFor(id).ClaimCgroupsUnified(key, plugin) + return o.mustOwnersFor(id).ClaimCgroupsUnified(key, plugin) } func (o *OwningPlugins) ClaimCgroupsPath(id, plugin string) error { - return o.OwnersFor(id).ClaimCgroupsPath(plugin) + return o.mustOwnersFor(id).ClaimCgroupsPath(plugin) } func (o *OwningPlugins) ClaimOomScoreAdj(id, plugin string) error { - return o.OwnersFor(id).ClaimOomScoreAdj(plugin) + return o.mustOwnersFor(id).ClaimOomScoreAdj(plugin) } func (o *OwningPlugins) ClaimRlimit(id, typ, plugin string) error { - return o.OwnersFor(id).ClaimRlimit(typ, plugin) + return o.mustOwnersFor(id).ClaimRlimit(typ, plugin) } func (o *OwningPlugins) ClearAnnotation(id, key, plugin string) { - o.OwnersFor(id).ClearAnnotation(key, plugin) + o.mustOwnersFor(id).ClearAnnotation(key, plugin) } func (o *OwningPlugins) ClearMount(id, key, plugin string) { - o.OwnersFor(id).ClearMount(key, plugin) + o.mustOwnersFor(id).ClearMount(key, plugin) } func (o *OwningPlugins) ClearDevice(id, key, plugin string) { - o.OwnersFor(id).ClearDevice(key, plugin) + o.mustOwnersFor(id).ClearDevice(key, plugin) } func (o *OwningPlugins) ClearEnv(id, key, plugin string) { - o.OwnersFor(id).ClearEnv(key, plugin) + o.mustOwnersFor(id).ClearEnv(key, plugin) } func (o *OwningPlugins) ClearArgs(id, plugin string) { - o.OwnersFor(id).ClearArgs(plugin) + o.mustOwnersFor(id).ClearArgs(plugin) } func (o *OwningPlugins) AnnotationOwner(id, key string) (string, bool) { - return o.OwnersFor(id).compoundOwner(Field_Annotations.Key(), key) + return o.ownersFor(id).compoundOwner(Field_Annotations.Key(), key) } func (o *OwningPlugins) MountOwner(id, destination string) (string, bool) { - return o.OwnersFor(id).compoundOwner(Field_Mounts.Key(), destination) + return o.ownersFor(id).compoundOwner(Field_Mounts.Key(), destination) } func (o *OwningPlugins) HooksOwner(id string) (string, bool) { - return o.OwnersFor(id).simpleOwner(Field_OciHooks.Key()) + return o.ownersFor(id).simpleOwner(Field_OciHooks.Key()) } func (o *OwningPlugins) DeviceOwner(id, path string) (string, bool) { - return o.OwnersFor(id).compoundOwner(Field_Devices.Key(), path) + return o.ownersFor(id).compoundOwner(Field_Devices.Key(), path) } func (o *OwningPlugins) EnvOwner(id, name string) (string, bool) { - return o.OwnersFor(id).compoundOwner(Field_Env.Key(), name) + return o.ownersFor(id).compoundOwner(Field_Env.Key(), name) } func (o *OwningPlugins) ArgsOwner(id string) (string, bool) { - return o.OwnersFor(id).simpleOwner(Field_Args.Key()) + return o.ownersFor(id).simpleOwner(Field_Args.Key()) } func (o *OwningPlugins) MemLimitOwner(id string) (string, bool) { - return o.OwnersFor(id).simpleOwner(Field_MemLimit.Key()) + return o.ownersFor(id).simpleOwner(Field_MemLimit.Key()) } func (o *OwningPlugins) MemReservationOwner(id string) (string, bool) { - return o.OwnersFor(id).simpleOwner(Field_MemReservation.Key()) + return o.ownersFor(id).simpleOwner(Field_MemReservation.Key()) } func (o *OwningPlugins) MemSwapLimitOwner(id string) (string, bool) { - return o.OwnersFor(id).simpleOwner(Field_MemSwapLimit.Key()) + return o.ownersFor(id).simpleOwner(Field_MemSwapLimit.Key()) } func (o *OwningPlugins) MemKernelLimitOwner(id string) (string, bool) { - return o.OwnersFor(id).simpleOwner(Field_MemKernelLimit.Key()) + return o.ownersFor(id).simpleOwner(Field_MemKernelLimit.Key()) } func (o *OwningPlugins) MemTCPLimitOwner(id string) (string, bool) { - return o.OwnersFor(id).simpleOwner(Field_MemTCPLimit.Key()) + return o.ownersFor(id).simpleOwner(Field_MemTCPLimit.Key()) } func (o *OwningPlugins) MemSwappinessOwner(id string) (string, bool) { - return o.OwnersFor(id).simpleOwner(Field_MemSwappiness.Key()) + return o.ownersFor(id).simpleOwner(Field_MemSwappiness.Key()) } func (o *OwningPlugins) MemDisableOomKillerOwner(id string) (string, bool) { - return o.OwnersFor(id).simpleOwner(Field_MemDisableOomKiller.Key()) + return o.ownersFor(id).simpleOwner(Field_MemDisableOomKiller.Key()) } func (o *OwningPlugins) MemUseHierarchyOwner(id string) (string, bool) { - return o.OwnersFor(id).simpleOwner(Field_MemUseHierarchy.Key()) + return o.ownersFor(id).simpleOwner(Field_MemUseHierarchy.Key()) } func (o *OwningPlugins) CPUSharesOwner(id string) (string, bool) { - return o.OwnersFor(id).simpleOwner(Field_CPUShares.Key()) + return o.ownersFor(id).simpleOwner(Field_CPUShares.Key()) } func (o *OwningPlugins) CPUQuotaOwner(id string) (string, bool) { - return o.OwnersFor(id).simpleOwner(Field_CPUQuota.Key()) + return o.ownersFor(id).simpleOwner(Field_CPUQuota.Key()) } func (o *OwningPlugins) CPUPeriodOwner(id string) (string, bool) { - return o.OwnersFor(id).simpleOwner(Field_CPUPeriod.Key()) + return o.ownersFor(id).simpleOwner(Field_CPUPeriod.Key()) } func (o *OwningPlugins) CPURealtimeRuntimeOwner(id string) (string, bool) { - return o.OwnersFor(id).simpleOwner(Field_CPURealtimeRuntime.Key()) + return o.ownersFor(id).simpleOwner(Field_CPURealtimeRuntime.Key()) } func (o *OwningPlugins) CPURealtimePeriodOwner(id string) (string, bool) { - return o.OwnersFor(id).simpleOwner(Field_CPURealtimePeriod.Key()) + return o.ownersFor(id).simpleOwner(Field_CPURealtimePeriod.Key()) } func (o *OwningPlugins) CPUSetCPUsOwner(id string) (string, bool) { - return o.OwnersFor(id).simpleOwner(Field_CPUSetCPUs.Key()) + return o.ownersFor(id).simpleOwner(Field_CPUSetCPUs.Key()) } func (o *OwningPlugins) CPUSetMemsOwner(id string) (string, bool) { - return o.OwnersFor(id).simpleOwner(Field_CPUSetMems.Key()) + return o.ownersFor(id).simpleOwner(Field_CPUSetMems.Key()) } func (o *OwningPlugins) PidsLimitOwner(id string) (string, bool) { - return o.OwnersFor(id).simpleOwner(Field_PidsLimit.Key()) + return o.ownersFor(id).simpleOwner(Field_PidsLimit.Key()) } func (o *OwningPlugins) HugepageLimitOwner(id, size string) (string, bool) { - return o.OwnersFor(id).compoundOwner(Field_HugepageLimits.Key(), size) + return o.ownersFor(id).compoundOwner(Field_HugepageLimits.Key(), size) } func (o *OwningPlugins) BlockioClassOwner(id string) (string, bool) { - return o.OwnersFor(id).simpleOwner(Field_BlockioClass.Key()) + return o.ownersFor(id).simpleOwner(Field_BlockioClass.Key()) } func (o *OwningPlugins) RdtClassOwner(id string) (string, bool) { - return o.OwnersFor(id).simpleOwner(Field_RdtClass.Key()) + return o.ownersFor(id).simpleOwner(Field_RdtClass.Key()) } func (o *OwningPlugins) CgroupsUnifiedOwner(id, key string) (string, bool) { - return o.OwnersFor(id).compoundOwner(Field_CgroupsUnified.Key(), key) + return o.ownersFor(id).compoundOwner(Field_CgroupsUnified.Key(), key) } func (o *OwningPlugins) CgroupsPathOwner(id string) (string, bool) { - return o.OwnersFor(id).simpleOwner(Field_CgroupsPath.Key()) + return o.ownersFor(id).simpleOwner(Field_CgroupsPath.Key()) } func (o *OwningPlugins) OomScoreAdjOwner(id string) (string, bool) { - return o.OwnersFor(id).simpleOwner(Field_OomScoreAdj.Key()) + return o.ownersFor(id).simpleOwner(Field_OomScoreAdj.Key()) } func (o *OwningPlugins) RlimitOwner(id, typ string) (string, bool) { - return o.OwnersFor(id).compoundOwner(Field_Rlimits.Key(), typ) + return o.ownersFor(id).compoundOwner(Field_Rlimits.Key(), typ) } -func (o *OwningPlugins) OwnersFor(id string) *FieldOwners { +func (o *OwningPlugins) mustOwnersFor(id string) *FieldOwners { f, ok := o.Owners[id] if !ok { f = NewFieldOwners() @@ -302,6 +302,14 @@ func (o *OwningPlugins) OwnersFor(id string) *FieldOwners { return f } +func (o *OwningPlugins) ownersFor(id string) *FieldOwners { + f, ok := o.Owners[id] + if !ok { + return nil + } + return f +} + func NewFieldOwners() *FieldOwners { return &FieldOwners{ Simple: make(map[int32]string), From 5a006df3c97bf22b03e80fbb133e243486b9c3e0 Mon Sep 17 00:00:00 2001 From: Krisztian Litkey Date: Sat, 17 May 2025 22:18:20 +0300 Subject: [PATCH 4/5] adaptation: add default validator tests. Add default validator tests to verify - proper rejection of OCI Hooks - proper validation of required plugins - allowed toleration of missing required plugins - proper validation of annotated extra required plugins Signed-off-by: Krisztian Litkey --- pkg/adaptation/adaptation_suite_test.go | 271 ++++++++++++++++++ pkg/adaptation/suite_test.go | 32 ++- .../default-validator_test.go | 256 +++++++++++++++++ 3 files changed, 552 insertions(+), 7 deletions(-) create mode 100644 plugins/default-validator/default-validator_test.go diff --git a/pkg/adaptation/adaptation_suite_test.go b/pkg/adaptation/adaptation_suite_test.go index 6dfcfb02..8d4f784a 100644 --- a/pkg/adaptation/adaptation_suite_test.go +++ b/pkg/adaptation/adaptation_suite_test.go @@ -36,6 +36,8 @@ import ( nri "github.com/containerd/nri/pkg/adaptation" "github.com/containerd/nri/pkg/api" + "github.com/containerd/nri/pkg/plugin" + validator "github.com/containerd/nri/plugins/default-validator/builtin" ) var _ = Describe("Configuration", func() { @@ -1015,6 +1017,275 @@ var _ = Describe("Plugin container creation adjustments", func() { ) }) + When("the default validator is enabled and OCI Hook injection is disabled", func() { + BeforeEach(func() { + s.Prepare( + &mockRuntime{ + options: []nri.Option{ + nri.WithDefaultValidator( + &validator.DefaultValidatorConfig{ + Enable: true, + RejectOCIHooks: true, + }, + ), + }, + }, + &mockPlugin{idx: "00", name: "foo"}, + &mockPlugin{idx: "10", name: "validator1"}, + &mockPlugin{idx: "20", name: "validator2"}, + ) + }) + + It("should reject OCI Hook injection", func() { + var ( + create = func(_ *mockPlugin, _ *api.PodSandbox, ctr *api.Container) (*api.ContainerAdjustment, []*api.ContainerUpdate, error) { + a := &api.ContainerAdjustment{} + if ctr.GetName() == "ctr1" { + a.AddHooks( + &api.Hooks{ + Prestart: []*api.Hook{ + { + Path: "/bin/sh", + Args: []string{"/bin/sh", "-c", "true"}, + }, + }, + }, + ) + } + + return a, nil, nil + } + + validate = func(_ *mockPlugin, _ *api.ValidateContainerAdjustmentRequest) error { + return nil + } + + runtime = s.runtime + plugins = s.plugins + ctx = context.Background() + + pod = &api.PodSandbox{ + Id: "pod0", + Name: "pod0", + Uid: "uid0", + Namespace: "default", + } + ctr0 = &api.Container{ + Id: "ctr0", + PodSandboxId: "pod0", + Name: "ctr0", + State: api.ContainerState_CONTAINER_CREATED, + } + ctr1 = &api.Container{ + Id: "ctr1", + PodSandboxId: "pod0", + Name: "ctr1", + State: api.ContainerState_CONTAINER_CREATED, + } + ) + + plugins[0].createContainer = create + plugins[1].validateAdjustment = validate + plugins[2].validateAdjustment = validate + + s.Startup() + podReq := &api.RunPodSandboxRequest{Pod: pod} + Expect(runtime.RunPodSandbox(ctx, podReq)).To(Succeed()) + + ctrReq := &api.CreateContainerRequest{ + Pod: pod, + Container: ctr0, + } + reply, err := runtime.CreateContainer(ctx, ctrReq) + Expect(reply).ToNot(BeNil()) + Expect(err).To(BeNil()) + + ctrReq = &api.CreateContainerRequest{ + Pod: pod, + Container: ctr1, + } + reply, err = runtime.CreateContainer(ctx, ctrReq) + Expect(err).ToNot(BeNil()) + Expect(reply).To(BeNil()) + }) + }) + + When("the default validator is enabled with some required plugins", func() { + const AnnotationDomain = plugin.AnnotationDomain + BeforeEach(func() { + s.Prepare( + &mockRuntime{ + options: []nri.Option{ + nri.WithDefaultValidator( + &validator.DefaultValidatorConfig{ + Enable: true, + RequiredPlugins: []string{ + "foo", + "bar", + }, + TolerateMissingAnnotation: "tolerate-missing-plugins." + AnnotationDomain, + }, + ), + }, + }, + &mockPlugin{idx: "00", name: "foo"}, + ) + }) + + It("should not allow container creation if required plugins are missing", func() { + var ( + runtime = s.runtime + ctx = context.Background() + + pod = &api.PodSandbox{ + Id: "pod0", + Name: "pod0", + Uid: "uid0", + Namespace: "default", + } + ) + + s.Startup() + podReq := &api.RunPodSandboxRequest{Pod: pod} + Expect(runtime.RunPodSandbox(ctx, podReq)).To(Succeed()) + + ctrReq := &api.CreateContainerRequest{ + Pod: pod, + Container: &api.Container{ + Id: "ctr0", + PodSandboxId: "pod0", + Name: "ctr0", + State: api.ContainerState_CONTAINER_CREATED, + }, + } + reply, err := runtime.CreateContainer(ctx, ctrReq) + Expect(reply).To(BeNil()) + Expect(err).ToNot(BeNil()) + }) + + It("should allow container creation, if missing plugins are tolerated", func() { + var ( + runtime = s.runtime + ctx = context.Background() + + pod = &api.PodSandbox{ + Id: "pod0", + Name: "pod0", + Uid: "uid0", + Namespace: "default", + Annotations: map[string]string{ + "tolerate-missing-plugins." + AnnotationDomain + "/container.ctr0": "true", + }, + } + ) + + s.Startup() + podReq := &api.RunPodSandboxRequest{Pod: pod} + Expect(runtime.RunPodSandbox(ctx, podReq)).To(Succeed()) + + ctrReq := &api.CreateContainerRequest{ + Pod: pod, + Container: &api.Container{ + Id: "ctr0", + PodSandboxId: "pod0", + Name: "ctr0", + State: api.ContainerState_CONTAINER_CREATED, + }, + } + reply, err := runtime.CreateContainer(ctx, ctrReq) + Expect(reply).ToNot(BeNil()) + Expect(err).To(BeNil()) + }) + + It("should allow container creation if all required plugins are present", func() { + var ( + runtime = s.runtime + ctx = context.Background() + + pod = &api.PodSandbox{ + Id: "pod0", + Name: "pod0", + Uid: "uid0", + Namespace: "default", + } + ) + + s.Startup() + podReq := &api.RunPodSandboxRequest{Pod: pod} + Expect(runtime.RunPodSandbox(ctx, podReq)).To(Succeed()) + + s.StartPlugins(&mockPlugin{idx: "10", name: "bar"}) + s.WaitForPluginsToSync(s.plugin("10-bar")) + + ctrReq := &api.CreateContainerRequest{ + Pod: pod, + Container: &api.Container{ + Id: "ctr0", + PodSandboxId: "pod0", + Name: "ctr0", + State: api.ContainerState_CONTAINER_CREATED, + }, + } + reply, err := runtime.CreateContainer(ctx, ctrReq) + Expect(reply).ToNot(BeNil()) + Expect(err).To(BeNil()) + }) + + It("should not allow container creation if annotated required plugins are missing", func() { + var ( + runtime = s.runtime + ctx = context.Background() + + pod = &api.PodSandbox{ + Id: "pod0", + Name: "pod0", + Uid: "uid0", + Namespace: "default", + Annotations: map[string]string{ + "required-plugins." + AnnotationDomain + "/container.ctr0": "[ \"xyzzy\" ]", + }, + } + ) + + s.Startup() + podReq := &api.RunPodSandboxRequest{Pod: pod} + Expect(runtime.RunPodSandbox(ctx, podReq)).To(Succeed()) + + s.StartPlugins(&mockPlugin{idx: "10", name: "bar"}) + s.WaitForPluginsToSync(s.plugin("10-bar")) + + ctrReq := &api.CreateContainerRequest{ + Pod: pod, + Container: &api.Container{ + Id: "ctr0", + PodSandboxId: "pod0", + Name: "ctr0", + State: api.ContainerState_CONTAINER_CREATED, + }, + } + reply, err := runtime.CreateContainer(ctx, ctrReq) + Expect(reply).To(BeNil()) + Expect(err).ToNot(BeNil()) + + s.StartPlugins(&mockPlugin{idx: "20", name: "xyzzy"}) + s.WaitForPluginsToSync(s.plugin("20-xyzzy")) + + ctrReq = &api.CreateContainerRequest{ + Pod: pod, + Container: &api.Container{ + Id: "ctr0", + PodSandboxId: "pod0", + Name: "ctr0", + State: api.ContainerState_CONTAINER_CREATED, + }, + } + reply, err = runtime.CreateContainer(ctx, ctrReq) + Expect(reply).ToNot(BeNil()) + Expect(err).To(BeNil()) + }) + + }) + }) // -------------------------------------------- diff --git a/pkg/adaptation/suite_test.go b/pkg/adaptation/suite_test.go index 8021ef6c..885113b9 100644 --- a/pkg/adaptation/suite_test.go +++ b/pkg/adaptation/suite_test.go @@ -53,6 +53,7 @@ type Suite struct { dir string // directory to create for test runtime *mockRuntime // runtime instance for test plugins []*mockPlugin // plugin instances for test + byName map[string]*mockPlugin } // SuiteOption can be applied to a suite. @@ -83,6 +84,10 @@ func (s *Suite) Prepare(runtime *mockRuntime, plugins ...*mockPlugin) string { s.runtime = runtime s.plugins = plugins + if s.byName == nil { + s.byName = make(map[string]*mockPlugin) + } + return dir } @@ -93,9 +98,11 @@ func (s *Suite) Dir() string { // Startup starts up the test suite. func (s *Suite) Startup() { + plugins := s.plugins + s.plugins = nil s.StartRuntime() - s.StartPlugins() - s.WaitForPluginsToSync() + s.StartPlugins(plugins...) + s.WaitForPluginsToSync(plugins...) } // StartRuntime starts the suite runtime. @@ -104,16 +111,18 @@ func (s *Suite) StartRuntime() { } // StartPlugins starts the suite plugins. -func (s *Suite) StartPlugins() { - for _, plugin := range s.plugins { +func (s *Suite) StartPlugins(plugins ...*mockPlugin) { + for _, plugin := range plugins { + s.plugins = append(s.plugins, plugin) + s.byName[plugin.FullName()] = plugin Expect(plugin.Start(s.dir)).To(Succeed()) } } -// WaitForPluginsToSync waits for the suite plugins to get synchronized. -func (s *Suite) WaitForPluginsToSync() { +// WaitForPluginsToSync waits for the given plugins to get synchronized. +func (s *Suite) WaitForPluginsToSync(plugins ...*mockPlugin) { timeout := time.After(startupTimeout) - for _, plugin := range s.plugins { + for _, plugin := range plugins { Expect(plugin.Wait(PluginSynchronized, timeout)).To(Succeed()) } } @@ -128,6 +137,11 @@ func (s *Suite) Cleanup() { Expect(os.RemoveAll(s.dir)).To(Succeed()) } +// Plugin returns a plugin started by StartPlugins by full plugin name. +func (s *Suite) plugin(fullName string) *mockPlugin { + return s.byName[fullName] +} + // ------------------------------------ func Log(format string, args ...interface{}) { @@ -521,6 +535,10 @@ func (m *mockPlugin) Stop() { m.q.Add(PluginStopped) } +func (m *mockPlugin) FullName() string { + return m.idx + "-" + m.name +} + func (m *mockPlugin) RuntimeName() string { return m.runtime } diff --git a/plugins/default-validator/default-validator_test.go b/plugins/default-validator/default-validator_test.go new file mode 100644 index 00000000..315e0777 --- /dev/null +++ b/plugins/default-validator/default-validator_test.go @@ -0,0 +1,256 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package validator + +import ( + "testing" + + "github.com/containerd/nri/pkg/api" + "github.com/stretchr/testify/require" +) + +func TestValidateReqiredPlugins(t *testing.T) { + type testCase struct { + name string + cfg *DefaultValidatorConfig + pod *api.PodSandbox + container *api.Container + plugins []*api.PluginInstance + fail bool + } + + for _, tc := range []*testCase{ + { + name: "no required plugins", + cfg: &DefaultValidatorConfig{ + Enable: true, + }, + pod: &api.PodSandbox{ + Id: "pod-id", + Name: "pod-name", + Namespace: "pod-namespace", + }, + container: &api.Container{ + Id: "container-id", + Name: "container-name", + }, + }, + { + name: "missing annotated required plugin", + cfg: &DefaultValidatorConfig{ + Enable: true, + }, + pod: &api.PodSandbox{ + Id: "pod-id", + Name: "pod-name", + Namespace: "pod-namespace", + Annotations: map[string]string{ + "required-plugins.noderesource.dev/container.container-name": "[ plugin ]", + }, + }, + container: &api.Container{ + Id: "container-id", + Name: "container-name", + }, + fail: true, + }, + { + name: "present annotated required plugin", + cfg: &DefaultValidatorConfig{ + Enable: true, + }, + pod: &api.PodSandbox{ + Id: "pod-id", + Name: "pod-name", + Namespace: "pod-namespace", + Annotations: map[string]string{ + "required-plugins.noderesource.dev/container.container-name": "[ plugin ]", + }, + }, + container: &api.Container{ + Id: "container-id", + Name: "container-name", + }, + plugins: []*api.PluginInstance{ + { + Name: "plugin", + Index: "00", + }, + }, + }, + + { + name: "missing global required plugin", + cfg: &DefaultValidatorConfig{ + Enable: true, + RequiredPlugins: []string{"plugin"}, + }, + pod: &api.PodSandbox{ + Id: "pod-id", + Name: "pod-name", + Namespace: "pod-namespace", + }, + container: &api.Container{ + Id: "container-id", + Name: "container-name", + }, + fail: true, + }, + { + name: "present global required plugin", + cfg: &DefaultValidatorConfig{ + Enable: true, + RequiredPlugins: []string{"plugin"}, + }, + pod: &api.PodSandbox{ + Id: "pod-id", + Name: "pod-name", + Namespace: "pod-namespace", + }, + container: &api.Container{ + Id: "container-id", + Name: "container-name", + }, + plugins: []*api.PluginInstance{ + { + Name: "plugin", + Index: "00", + }, + }, + }, + { + name: "tolerated missing (global required) plugin", + cfg: &DefaultValidatorConfig{ + Enable: true, + RequiredPlugins: []string{"plugin"}, + TolerateMissingAnnotation: "tolerate-missing-plugins", + }, + pod: &api.PodSandbox{ + Id: "pod-id", + Name: "pod-name", + Namespace: "pod-namespace", + Annotations: map[string]string{ + "tolerate-missing-plugins": "true", + }, + }, + container: &api.Container{ + Id: "container-id", + Name: "container-name", + }, + }, + { + name: "present annotated and global required plugin", + cfg: &DefaultValidatorConfig{ + Enable: true, + RequiredPlugins: []string{"plugin1"}, + }, + pod: &api.PodSandbox{ + Id: "pod-id", + Name: "pod-name", + Namespace: "pod-namespace", + Annotations: map[string]string{ + "required-plugins.noderesource.dev/container.container-name": "[ plugin2 ]", + }, + }, + container: &api.Container{ + Id: "container-id", + Name: "container-name", + }, + plugins: []*api.PluginInstance{ + { + Name: "plugin1", + Index: "00", + }, + { + Name: "plugin2", + Index: "01", + }, + }, + }, + { + name: "missing annotated with present global required plugin", + cfg: &DefaultValidatorConfig{ + Enable: true, + RequiredPlugins: []string{"plugin1"}, + }, + pod: &api.PodSandbox{ + Id: "pod-id", + Name: "pod-name", + Namespace: "pod-namespace", + Annotations: map[string]string{ + "required-plugins.noderesource.dev/container.container-name": "[ plugin2 ]", + }, + }, + container: &api.Container{ + Id: "container-id", + Name: "container-name", + }, + plugins: []*api.PluginInstance{ + { + Name: "plugin1", + Index: "00", + }, + }, + fail: true, + }, + { + name: "present annotated with missing global required plugin", + cfg: &DefaultValidatorConfig{ + Enable: true, + RequiredPlugins: []string{"plugin1"}, + }, + pod: &api.PodSandbox{ + Id: "pod-id", + Name: "pod-name", + Namespace: "pod-namespace", + Annotations: map[string]string{ + "required-plugins.noderesource.dev/container.container-name": "[ plugin2 ]", + }, + }, + container: &api.Container{ + Id: "container-id", + Name: "container-name", + }, + plugins: []*api.PluginInstance{ + { + Name: "plugin2", + Index: "00", + }, + }, + fail: true, + }, + } { + t.Run(tc.name, func(t *testing.T) { + var ( + v = NewDefaultValidator(tc.cfg) + req = &api.ValidateContainerAdjustmentRequest{ + Pod: tc.pod, + Container: tc.container, + Plugins: tc.plugins, + } + ) + + err := v.validateRequiredPlugins(req) + if tc.fail { + require.Error(t, err) + } else { + require.NoError(t, err) + } + }) + } + +} From 00d85a9b848fb94041fd9def8700c671add26fc9 Mon Sep 17 00:00:00 2001 From: Krisztian Litkey Date: Mon, 19 May 2025 17:35:40 +0300 Subject: [PATCH 5/5] README.md: add a brief description of validation. Add a section to the documentation briefly describing the core ideas and concepts of pluggable validation and the feature set provided by the default builtin validator. Co-authored-by: Chris Henzie Signed-off-by: Krisztian Litkey --- README.md | 79 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 79 insertions(+) diff --git a/README.md b/README.md index 6ecebcae..e37085ea 100644 --- a/README.md +++ b/README.md @@ -83,6 +83,11 @@ provides functions for - hooking the plugin into pod/container lifecycle events - shutting down the plugin +An additional interface is provided for validating the changes active plugins +have requested to containers. This interface allows one to set up and enforce +cluster- or node-wide boundary conditions for changes NRI plugins are allowed +to make. + ### Plugin Registration Before a plugin can start receiving and processing container events, it needs @@ -277,6 +282,80 @@ can be updated this way: - Block I/O class - RDT class +### Container Adjustment Validation + +NRI plugins operate as trusted extensions of the container runtime, granting +them significant privileges to alter container specs. While this extensibility +is powerful with valid use cases, some of the capabilities granted to plugins +allow modifying security-sensitive settings of containers. As such they also +come with the risk that a plugin could inadvertently or maliciously weaken a +container's isolation or security posture, potentially overriding policies set +by cluster orchestrators such as K8s. + +NRI offers cluster administrators a mechanism to exercise fine-grained control +over what changes plugins are allowed to make to containers, allowing cluster +administrators to lock down selected features in NRI or allowing them to only +be used a subset of plugins. Changes in NRI are made in two phases: “Mutating” +plugins propose changes, and “Validating” plugins approve or deny them. + +Validating plugins are invoked during container creation after all the changes +requested to containers have been collected. Validating plugins receive the +changes with extra information about which of the plugins requested what +changes. They can then choose to reject the changes if they violate some of the +conditions being validated. + +Validation has transactional semantics. If any validating plugin rejects an +adjustment, creation of the adjusted container will fail and none of the other +related changes will be made. + +#### Validation Use Cases + +Some key validation uses cases include + +1. Functional Validators: These plugins care about the final state and +consistency. They check if the combined effect of all mutations result +in a valid configuration (e.g. are the resource limits sane). + +2. Security Validators: These plugins are interested in which plugin is +attempting to modify sensitive fields. They use the extra data passed to +plugins in addition to adjustments to check if a potentially untrusted +plugin tried to modify a restricted field, regardless of the value. +Rejection might occur simply because a non-approved plugin touched a +specific field. Plugins like this may need to be assured to run, and to +have workloads fail-closed if the validator is not running. + +3. Mandatory Plugin Validators: These ensure that specific plugins, required +for certain workloads have successfully run. They might use the extra metadata +passed to validator in addition to adjustments to confirm the mandatory +plugin owns certain critical fields and potentially use the list of plugins +that processed the container to ensure all mandatory plugins were consulted. + +#### Default Validation + +The default built-in validator plugin provides configurable minimal validation. +It is disabled by default. It can be enabled and selectively configured to + +1. Reject OCI Hook injection: Reject any adjustment which tries to inject +OCI Hooks into a container. + +2. Verify global mandatory plugins: Verify that all configured mandatory +plugins are present and have processed a container. Otherwise reject the +creation of the container. + +3. Verify annotated mandatory plugins: Verify that an annotated set of +container-specific mandatory plugins are present and have processed a +container. Otherwise reject the creation of the container. + +Containers can be annotated to tolerate missing required plugins. This +allows one to deploy mandatory plugins as containers themselves. + +#### Default Validation Scope + +Currently only OCI hook injection can be restricted using the default +validator. However, this probably will change in the future. Especially +when NRI is extended with control over new container parameters. If such +parameters will have security implications, corresponding configurable +restrictions will be introduced to the default validator. ## Runtime Adaptation