From 2ae27e37b8c56b407e153107718db90c72b05f09 Mon Sep 17 00:00:00 2001 From: Mathieu Grzybek Date: Tue, 7 Jul 2026 16:59:55 +0200 Subject: [PATCH 01/27] go(init): scaffolding the project using kubebuilder Signed-off-by: Mathieu Grzybek --- .custom-gcl.yml | 11 + .devcontainer/devcontainer.json | 35 ++ .devcontainer/post-install.sh | 153 ++++++++ .github/workflows/lint.yml | 29 ++ .github/workflows/test-e2e.yml | 38 ++ .github/workflows/test.yml | 29 ++ .golangci.yml | 69 ++++ Makefile | 259 +++++++++++++ PROJECT | 11 + cmd/main.go | 205 +++++++++++ .../default/cert_metrics_manager_patch.yaml | 30 ++ config/default/kustomization.yaml | 234 ++++++++++++ config/default/manager_metrics_patch.yaml | 4 + config/default/metrics_service.yaml | 18 + config/manager/kustomization.yaml | 2 + config/manager/manager.yaml | 102 ++++++ .../network-policy/allow-metrics-traffic.yaml | 27 ++ config/network-policy/kustomization.yaml | 2 + config/prometheus/kustomization.yaml | 11 + config/prometheus/monitor.yaml | 27 ++ config/prometheus/monitor_tls_patch.yaml | 19 + config/rbac/kustomization.yaml | 20 + config/rbac/leader_election_role.yaml | 40 ++ config/rbac/leader_election_role_binding.yaml | 15 + config/rbac/metrics_auth_role.yaml | 17 + config/rbac/metrics_auth_role_binding.yaml | 12 + config/rbac/metrics_reader_role.yaml | 9 + config/rbac/role.yaml | 11 + config/rbac/role_binding.yaml | 15 + config/rbac/service_account.yaml | 8 + go.mod | 104 ++++++ go.sum | 265 ++++++++++++++ hack/boilerplate.go.txt | 15 + .../controller/openstackcluster_controller.go | 276 ++++++++++++++ internal/openstack/cloud.go | 317 ++++++++++++++++ internal/openstack/purge.go | 68 ++++ internal/openstack/resources.go | 341 ++++++++++++++++++ internal/openstack/tls.go | 19 + test/e2e/e2e_suite_test.go | 119 ++++++ test/e2e/e2e_test.go | 339 +++++++++++++++++ test/utils/utils.go | 226 ++++++++++++ 41 files changed, 3551 insertions(+) create mode 100644 .custom-gcl.yml create mode 100644 .devcontainer/devcontainer.json create mode 100644 .devcontainer/post-install.sh create mode 100644 .github/workflows/lint.yml create mode 100644 .github/workflows/test-e2e.yml create mode 100644 .github/workflows/test.yml create mode 100644 .golangci.yml create mode 100644 Makefile create mode 100644 PROJECT create mode 100644 cmd/main.go create mode 100644 config/default/cert_metrics_manager_patch.yaml create mode 100644 config/default/kustomization.yaml create mode 100644 config/default/manager_metrics_patch.yaml create mode 100644 config/default/metrics_service.yaml create mode 100644 config/manager/kustomization.yaml create mode 100644 config/manager/manager.yaml create mode 100644 config/network-policy/allow-metrics-traffic.yaml create mode 100644 config/network-policy/kustomization.yaml create mode 100644 config/prometheus/kustomization.yaml create mode 100644 config/prometheus/monitor.yaml create mode 100644 config/prometheus/monitor_tls_patch.yaml create mode 100644 config/rbac/kustomization.yaml create mode 100644 config/rbac/leader_election_role.yaml create mode 100644 config/rbac/leader_election_role_binding.yaml create mode 100644 config/rbac/metrics_auth_role.yaml create mode 100644 config/rbac/metrics_auth_role_binding.yaml create mode 100644 config/rbac/metrics_reader_role.yaml create mode 100644 config/rbac/role.yaml create mode 100644 config/rbac/role_binding.yaml create mode 100644 config/rbac/service_account.yaml create mode 100644 go.mod create mode 100644 go.sum create mode 100644 hack/boilerplate.go.txt create mode 100644 internal/controller/openstackcluster_controller.go create mode 100644 internal/openstack/cloud.go create mode 100644 internal/openstack/purge.go create mode 100644 internal/openstack/resources.go create mode 100644 internal/openstack/tls.go create mode 100644 test/e2e/e2e_suite_test.go create mode 100644 test/e2e/e2e_test.go create mode 100644 test/utils/utils.go diff --git a/.custom-gcl.yml b/.custom-gcl.yml new file mode 100644 index 0000000..d9ae33b --- /dev/null +++ b/.custom-gcl.yml @@ -0,0 +1,11 @@ +# This file configures golangci-lint with module plugins. +# When you run 'make lint', it will automatically build a custom golangci-lint binary +# with all the plugins listed below. +# +# See: https://golangci-lint.run/plugins/module-plugins/ +version: v2.12.2 +plugins: + # logcheck validates structured logging calls and parameters (e.g., balanced key-value pairs) + - module: "sigs.k8s.io/logtools" + import: "sigs.k8s.io/logtools/logcheck/gclplugin" + version: latest diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 0000000..a96838b --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,35 @@ +{ + "name": "Kubebuilder DevContainer", + "image": "golang:1.26", + "features": { + "ghcr.io/devcontainers/features/docker-in-docker:2": { + "moby": false, + "dockerDefaultAddressPool": "base=172.30.0.0/16,size=24" + }, + "ghcr.io/devcontainers/features/git:1": {}, + "ghcr.io/devcontainers/features/common-utils:2": { + "upgradePackages": true + } + }, + + "runArgs": ["--privileged", "--init"], + + "customizations": { + "vscode": { + "settings": { + "terminal.integrated.shell.linux": "/bin/bash" + }, + "extensions": [ + "ms-kubernetes-tools.vscode-kubernetes-tools", + "ms-azuretools.vscode-docker" + ] + } + }, + + "remoteEnv": { + "GO111MODULE": "on" + }, + + "onCreateCommand": "bash .devcontainer/post-install.sh" +} + diff --git a/.devcontainer/post-install.sh b/.devcontainer/post-install.sh new file mode 100644 index 0000000..6d75a49 --- /dev/null +++ b/.devcontainer/post-install.sh @@ -0,0 +1,153 @@ +#!/bin/bash +set -euo pipefail + +echo "====================================" +echo "Kubebuilder DevContainer Setup" +echo "====================================" + +# Verify running as root (required for installing to /usr/local/bin and /etc) +if [ "$(id -u)" -ne 0 ]; then + echo "ERROR: This script must be run as root" + exit 1 +fi + +echo "" +echo "Detecting system architecture..." +# Detect architecture using uname +MACHINE=$(uname -m) +case "${MACHINE}" in + x86_64) + ARCH="amd64" + ;; + aarch64|arm64) + ARCH="arm64" + ;; + *) + echo "WARNING: Unsupported architecture ${MACHINE}, defaulting to amd64" + ARCH="amd64" + ;; +esac +echo "Architecture: ${ARCH}" + +echo "" +echo "------------------------------------" +echo "Setting up bash completion..." +echo "------------------------------------" + +BASH_COMPLETIONS_DIR="/usr/share/bash-completion/completions" + +# Enable bash-completion in root's .bashrc (devcontainer runs as root) +if ! grep -q "source /usr/share/bash-completion/bash_completion" ~/.bashrc 2>/dev/null; then + echo 'source /usr/share/bash-completion/bash_completion' >> ~/.bashrc + echo "Added bash-completion to .bashrc" +fi + +echo "" +echo "------------------------------------" +echo "Installing development tools..." +echo "------------------------------------" + +# Install kind +if ! command -v kind &> /dev/null; then + echo "Installing kind..." + curl -Lo /usr/local/bin/kind "https://kind.sigs.k8s.io/dl/latest/kind-linux-${ARCH}" + chmod +x /usr/local/bin/kind + echo "kind installed successfully" +fi + +# Generate kind bash completion +if command -v kind &> /dev/null; then + if kind completion bash > "${BASH_COMPLETIONS_DIR}/kind" 2>/dev/null; then + echo "kind completion installed" + else + echo "WARNING: Failed to generate kind completion" + fi +fi + +# Install kubebuilder +if ! command -v kubebuilder &> /dev/null; then + echo "Installing kubebuilder..." + curl -Lo /usr/local/bin/kubebuilder "https://go.kubebuilder.io/dl/latest/linux/${ARCH}" + chmod +x /usr/local/bin/kubebuilder + echo "kubebuilder installed successfully" +fi + +# Generate kubebuilder bash completion +if command -v kubebuilder &> /dev/null; then + if kubebuilder completion bash > "${BASH_COMPLETIONS_DIR}/kubebuilder" 2>/dev/null; then + echo "kubebuilder completion installed" + else + echo "WARNING: Failed to generate kubebuilder completion" + fi +fi + +# Install kubectl +if ! command -v kubectl &> /dev/null; then + echo "Installing kubectl..." + KUBECTL_VERSION=$(curl -Ls https://dl.k8s.io/release/stable.txt) + curl -Lo /usr/local/bin/kubectl "https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/${ARCH}/kubectl" + chmod +x /usr/local/bin/kubectl + echo "kubectl installed successfully" +fi + +# Generate kubectl bash completion +if command -v kubectl &> /dev/null; then + if kubectl completion bash > "${BASH_COMPLETIONS_DIR}/kubectl" 2>/dev/null; then + echo "kubectl completion installed" + else + echo "WARNING: Failed to generate kubectl completion" + fi +fi + +# Generate Docker bash completion +if command -v docker &> /dev/null; then + if docker completion bash > "${BASH_COMPLETIONS_DIR}/docker" 2>/dev/null; then + echo "docker completion installed" + else + echo "WARNING: Failed to generate docker completion" + fi +fi + +echo "" +echo "------------------------------------" +echo "Configuring Docker environment..." +echo "------------------------------------" + +# Wait for Docker to be ready +echo "Waiting for Docker to be ready..." +for i in {1..30}; do + if docker info >/dev/null 2>&1; then + echo "Docker is ready" + break + fi + if [ "$i" -eq 30 ]; then + echo "WARNING: Docker not ready after 30s" + fi + sleep 1 +done + +# Create kind network (ignore if already exists) +if ! docker network inspect kind >/dev/null 2>&1; then + if docker network create kind >/dev/null 2>&1; then + echo "Created kind network" + else + echo "WARNING: Failed to create kind network (may already exist)" + fi +fi + +echo "" +echo "------------------------------------" +echo "Verifying installations..." +echo "------------------------------------" +kind version +kubebuilder version +kubectl version --client +docker --version +go version + +echo "" +echo "====================================" +echo "DevContainer ready!" +echo "====================================" +echo "All development tools installed successfully." +echo "You can now start building Kubernetes operators." diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 0000000..d3f5b7b --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,29 @@ +name: Lint + +on: + push: + pull_request: + +permissions: {} + +jobs: + lint: + permissions: + contents: read + name: Run on Ubuntu + runs-on: ubuntu-latest + steps: + - name: Clone the code + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - name: Setup Go + uses: actions/setup-go@4b73464bb391d4059bd26b0524d20df3927bd417 # v6.3.0 + with: + go-version-file: go.mod + + - name: Check linter configuration + run: make lint-config + - name: Run linter + run: make lint diff --git a/.github/workflows/test-e2e.yml b/.github/workflows/test-e2e.yml new file mode 100644 index 0000000..83e295e --- /dev/null +++ b/.github/workflows/test-e2e.yml @@ -0,0 +1,38 @@ +name: E2E Tests + +on: + push: + pull_request: + +permissions: {} + +jobs: + test-e2e: + permissions: + contents: read + name: Run on Ubuntu + runs-on: ubuntu-latest + steps: + - name: Clone the code + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - name: Setup Go + uses: actions/setup-go@4b73464bb391d4059bd26b0524d20df3927bd417 # v6.3.0 + with: + go-version-file: go.mod + + - name: Install the latest version of kind + run: | + curl -Lo ./kind https://kind.sigs.k8s.io/dl/latest/kind-linux-$(go env GOARCH) + chmod +x ./kind + sudo mv ./kind /usr/local/bin/kind + + - name: Verify kind installation + run: kind version + + - name: Running Test e2e + run: | + go mod tidy + make test-e2e diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..27e82c8 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,29 @@ +name: Tests + +on: + push: + pull_request: + +permissions: {} + +jobs: + test: + permissions: + contents: read + name: Run on Ubuntu + runs-on: ubuntu-latest + steps: + - name: Clone the code + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - name: Setup Go + uses: actions/setup-go@4b73464bb391d4059bd26b0524d20df3927bd417 # v6.3.0 + with: + go-version-file: go.mod + + - name: Running Tests + run: | + go mod tidy + make test diff --git a/.golangci.yml b/.golangci.yml new file mode 100644 index 0000000..b139f79 --- /dev/null +++ b/.golangci.yml @@ -0,0 +1,69 @@ +version: "2" +run: + allow-parallel-runners: true +linters: + default: none + enable: + - copyloopvar + - depguard + - dupl + - errcheck + - ginkgolinter + - goconst + - gocyclo + - govet + - ineffassign + - lll + - modernize + - misspell + - nakedret + - prealloc + - revive + - staticcheck + - unconvert + - unparam + - unused + - logcheck + settings: + custom: + logcheck: + type: "module" + description: Checks Go logging calls for Kubernetes logging conventions. + depguard: + rules: + forbid-sort-pkg: + deny: + - pkg: sort + desc: Should be replaced with slices package + revive: + rules: + - name: comment-spacings + - name: import-shadowing + modernize: + disable: + - omitzero + - newexpr + exclusions: + generated: lax + rules: + - linters: + - lll + path: api/* + - linters: + - dupl + - lll + path: internal/* + paths: + - third_party$ + - builtin$ + - examples$ +formatters: + enable: + - gofmt + - goimports + exclusions: + generated: lax + paths: + - third_party$ + - builtin$ + - examples$ diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..15b2f2a --- /dev/null +++ b/Makefile @@ -0,0 +1,259 @@ +# Image URL to use all building/pushing image targets +IMG ?= controller:latest +# YEAR defines the year value used for substituting the YEAR placeholder in the boilerplate header. +YEAR ?= $(shell date +%Y) + +# Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set) +ifeq (,$(shell go env GOBIN)) +GOBIN=$(shell go env GOPATH)/bin +else +GOBIN=$(shell go env GOBIN) +endif + +# CONTAINER_TOOL defines the container tool to be used for building images. +# Be aware that the target commands are only tested with Docker which is +# scaffolded by default. However, you might want to replace it to use other +# tools. (i.e. podman) +CONTAINER_TOOL ?= docker + +# Setting SHELL to bash allows bash commands to be executed by recipes. +# Options are set to exit when a recipe line exits non-zero or a piped command fails. +SHELL = /usr/bin/env bash -o pipefail +.SHELLFLAGS = -ec + +.PHONY: all +all: build + +##@ General + +# The help target prints out all targets with their descriptions organized +# beneath their categories. The categories are represented by '##@' and the +# target descriptions by '##'. The awk command is responsible for reading the +# entire set of makefiles included in this invocation, looking for lines of the +# file as xyz: ## something, and then pretty-format the target and help. Then, +# if there's a line with ##@ something, that gets pretty-printed as a category. +# More info on the usage of ANSI control characters for terminal formatting: +# https://en.wikipedia.org/wiki/ANSI_escape_code#SGR_parameters +# More info on the awk command: +# http://linuxcommand.org/lc3_adv_awk.php + +.PHONY: help +help: ## Display this help. + @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf " \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST) + +##@ Development + +.PHONY: manifests +manifests: controller-gen ## Generate WebhookConfiguration, ClusterRole and CustomResourceDefinition objects. + "$(CONTROLLER_GEN)" rbac:roleName=manager-role crd webhook paths="./..." output:crd:artifacts:config=config/crd/bases + +.PHONY: generate +generate: controller-gen ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations. + "$(CONTROLLER_GEN)" object:headerFile="hack/boilerplate.go.txt",year=$(YEAR) paths="./..." + +.PHONY: fmt +fmt: ## Run go fmt against code. + go fmt ./... + +.PHONY: vet +vet: ## Run go vet against code. + go vet ./... + +.PHONY: test +test: manifests generate fmt vet setup-envtest ## Run tests. + KUBEBUILDER_ASSETS="$(shell "$(ENVTEST)" use $(ENVTEST_K8S_VERSION) --bin-dir "$(LOCALBIN)" -p path)" go test $$(go list ./... | grep -v /e2e) -coverprofile cover.out + +# TODO(user): To use a different vendor for e2e tests, modify the setup under 'tests/e2e'. +# The default setup assumes Kind is pre-installed and builds/loads the Manager Docker image locally. +# kubectl kuberc is disabled by default for test isolation; enable with: +# - KUBECTL_KUBERC=true +# CertManager is installed by default; skip with: +# - CERT_MANAGER_INSTALL_SKIP=true +KIND_CLUSTER ?= cluster-api-janitor-openstack-test-e2e + +.PHONY: setup-test-e2e +setup-test-e2e: ## Set up a Kind cluster for e2e tests if it does not exist + @command -v $(KIND) >/dev/null 2>&1 || { \ + echo "Kind is not installed. Please install Kind manually."; \ + exit 1; \ + } + @case "$$($(KIND) get clusters)" in \ + *"$(KIND_CLUSTER)"*) \ + echo "Kind cluster '$(KIND_CLUSTER)' already exists. Skipping creation." ;; \ + *) \ + echo "Creating Kind cluster '$(KIND_CLUSTER)'..."; \ + $(KIND) create cluster --name $(KIND_CLUSTER) ;; \ + esac + +.PHONY: test-e2e +test-e2e: setup-test-e2e manifests generate fmt vet ## Run the e2e tests. Expected an isolated environment using Kind. + KIND=$(KIND) KIND_CLUSTER=$(KIND_CLUSTER) go test -tags=e2e ./test/e2e/ -v -ginkgo.v + $(MAKE) cleanup-test-e2e + +.PHONY: cleanup-test-e2e +cleanup-test-e2e: ## Tear down the Kind cluster used for e2e tests + @$(KIND) delete cluster --name $(KIND_CLUSTER) + +.PHONY: lint +lint: golangci-lint ## Run golangci-lint linter + "$(GOLANGCI_LINT)" run + +.PHONY: lint-fix +lint-fix: golangci-lint ## Run golangci-lint linter and perform fixes + "$(GOLANGCI_LINT)" run --fix + +.PHONY: lint-config +lint-config: golangci-lint ## Verify golangci-lint linter configuration + "$(GOLANGCI_LINT)" config verify + +##@ Build + +.PHONY: build +build: manifests generate fmt vet ## Build manager binary. + go build -o bin/manager cmd/main.go + +.PHONY: run +run: manifests generate fmt vet ## Run a controller from your host. + go run ./cmd/main.go + +# If you wish to build the manager image targeting other platforms you can use the --platform flag. +# (i.e. docker build --platform linux/arm64). However, you must enable docker buildKit for it. +# More info: https://docs.docker.com/develop/develop-images/build_enhancements/ +.PHONY: docker-build +docker-build: ## Build docker image with the manager. + $(CONTAINER_TOOL) build -t ${IMG} . + +.PHONY: docker-push +docker-push: ## Push docker image with the manager. + $(CONTAINER_TOOL) push ${IMG} + +# PLATFORMS defines the target platforms for the manager image be built to provide support to multiple +# architectures. (i.e. make docker-buildx IMG=myregistry/mypoperator:0.0.1). To use this option you need to: +# - be able to use docker buildx. More info: https://docs.docker.com/build/buildx/ +# - have enabled BuildKit. More info: https://docs.docker.com/develop/develop-images/build_enhancements/ +# - be able to push the image to your registry (i.e. if you do not set a valid value via IMG=> then the export will fail) +# To adequately provide solutions that are compatible with multiple platforms, you should consider using this option. +PLATFORMS ?= linux/arm64,linux/amd64,linux/s390x,linux/ppc64le +.PHONY: docker-buildx +docker-buildx: ## Build and push docker image for the manager for cross-platform support + # copy existing Dockerfile and insert --platform=${BUILDPLATFORM} into Dockerfile.cross, and preserve the original Dockerfile + sed -e '1 s/\(^FROM\)/FROM --platform=\$$\{BUILDPLATFORM\}/; t' -e ' 1,// s//FROM --platform=\$$\{BUILDPLATFORM\}/' Dockerfile > Dockerfile.cross + - $(CONTAINER_TOOL) buildx create --name cluster-api-janitor-openstack-builder + $(CONTAINER_TOOL) buildx use cluster-api-janitor-openstack-builder + - $(CONTAINER_TOOL) buildx build --push --platform=$(PLATFORMS) --tag ${IMG} -f Dockerfile.cross . + - $(CONTAINER_TOOL) buildx rm cluster-api-janitor-openstack-builder + rm Dockerfile.cross + +.PHONY: build-installer +build-installer: manifests generate kustomize ## Generate a consolidated YAML with CRDs and deployment. + mkdir -p dist + cd config/manager && "$(KUSTOMIZE)" edit set image controller=${IMG} + "$(KUSTOMIZE)" build config/default > dist/install.yaml + +##@ Deployment + +ifndef ignore-not-found + ignore-not-found = false +endif + +.PHONY: install +install: manifests kustomize ## Install CRDs into the K8s cluster specified in ~/.kube/config. + @out="$$( "$(KUSTOMIZE)" build config/crd 2>/dev/null || true )"; \ + if [ -n "$$out" ]; then echo "$$out" | "$(KUBECTL)" apply -f -; else echo "No CRDs to install; skipping."; fi + +.PHONY: uninstall +uninstall: manifests kustomize ## Uninstall CRDs from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion. + @out="$$( "$(KUSTOMIZE)" build config/crd 2>/dev/null || true )"; \ + if [ -n "$$out" ]; then echo "$$out" | "$(KUBECTL)" delete --ignore-not-found=$(ignore-not-found) -f -; else echo "No CRDs to delete; skipping."; fi + +.PHONY: deploy +deploy: manifests kustomize ## Deploy controller to the K8s cluster specified in ~/.kube/config. + cd config/manager && "$(KUSTOMIZE)" edit set image controller=${IMG} + "$(KUSTOMIZE)" build config/default | "$(KUBECTL)" apply -f - + +.PHONY: undeploy +undeploy: kustomize ## Undeploy controller from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion. + "$(KUSTOMIZE)" build config/default | "$(KUBECTL)" delete --ignore-not-found=$(ignore-not-found) -f - + +##@ Dependencies + +## Location to install dependencies to +LOCALBIN ?= $(shell pwd)/bin +$(LOCALBIN): + mkdir -p "$(LOCALBIN)" + +## Tool Binaries +KUBECTL ?= kubectl +KIND ?= kind +KUSTOMIZE ?= $(LOCALBIN)/kustomize +CONTROLLER_GEN ?= $(LOCALBIN)/controller-gen +ENVTEST ?= $(LOCALBIN)/setup-envtest +GOLANGCI_LINT = $(LOCALBIN)/golangci-lint + +## Tool Versions +KUSTOMIZE_VERSION ?= v5.8.1 +CONTROLLER_TOOLS_VERSION ?= v0.21.0 + +#ENVTEST_VERSION is the controller-runtime version to use for setup-envtest, derived from go.mod +ENVTEST_VERSION ?= $(shell v='$(call gomodver,sigs.k8s.io/controller-runtime)'; \ + [ -n "$$v" ] || { echo "Set ENVTEST_VERSION manually (controller-runtime replace has no tag)" >&2; exit 1; }; \ + printf '%s\n' "$$v") + +#ENVTEST_K8S_VERSION is the version of Kubernetes to use for setting up ENVTEST binaries (i.e. 1.31) +ENVTEST_K8S_VERSION ?= $(shell v='$(call gomodver,k8s.io/api)'; \ + [ -n "$$v" ] || { echo "Set ENVTEST_K8S_VERSION manually (k8s.io/api replace has no tag)" >&2; exit 1; }; \ + printf '%s\n' "$$v" | sed -E 's/^v?[0-9]+\.([0-9]+).*/1.\1/') + +GOLANGCI_LINT_VERSION ?= v2.12.2 +.PHONY: kustomize +kustomize: $(KUSTOMIZE) ## Download kustomize locally if necessary. +$(KUSTOMIZE): $(LOCALBIN) + $(call go-install-tool,$(KUSTOMIZE),sigs.k8s.io/kustomize/kustomize/v5,$(KUSTOMIZE_VERSION)) + +.PHONY: controller-gen +controller-gen: $(CONTROLLER_GEN) ## Download controller-gen locally if necessary. +$(CONTROLLER_GEN): $(LOCALBIN) + $(call go-install-tool,$(CONTROLLER_GEN),sigs.k8s.io/controller-tools/cmd/controller-gen,$(CONTROLLER_TOOLS_VERSION)) + +.PHONY: setup-envtest +setup-envtest: envtest ## Download the binaries required for ENVTEST in the local bin directory. + @echo "Setting up envtest binaries for Kubernetes version $(ENVTEST_K8S_VERSION)..." + @"$(ENVTEST)" use $(ENVTEST_K8S_VERSION) --bin-dir "$(LOCALBIN)" -p path || { \ + echo "Error: Failed to set up envtest binaries for version $(ENVTEST_K8S_VERSION)."; \ + exit 1; \ + } + +.PHONY: envtest +envtest: $(ENVTEST) ## Download setup-envtest locally if necessary. +$(ENVTEST): $(LOCALBIN) + $(call go-install-tool,$(ENVTEST),sigs.k8s.io/controller-runtime/tools/setup-envtest,$(ENVTEST_VERSION)) + +.PHONY: golangci-lint +golangci-lint: $(GOLANGCI_LINT) ## Download golangci-lint locally if necessary. +$(GOLANGCI_LINT): $(LOCALBIN) + $(call go-install-tool,$(GOLANGCI_LINT),github.com/golangci/golangci-lint/v2/cmd/golangci-lint,$(GOLANGCI_LINT_VERSION)) + @test -f .custom-gcl.yml && { \ + echo "Building custom golangci-lint with plugins..." && \ + $(GOLANGCI_LINT) custom --destination $(LOCALBIN) --name golangci-lint-custom && \ + mv -f $(LOCALBIN)/golangci-lint-custom $(GOLANGCI_LINT); \ + } || true + +# go-install-tool will 'go install' any package with custom target and name of binary, if it doesn't exist +# $1 - target path with name of binary +# $2 - package url which can be installed +# $3 - specific version of package +define go-install-tool +@[ -f "$(1)-$(3)" ] && [ "$$(readlink -- "$(1)" 2>/dev/null)" = "$(1)-$(3)" ] || { \ +set -e; \ +package=$(2)@$(3) ;\ +echo "Downloading $${package}" ;\ +rm -f "$(1)" ;\ +GOBIN="$(LOCALBIN)" go install $${package} ;\ +mv "$(LOCALBIN)/$$(basename "$(1)")" "$(1)-$(3)" ;\ +} ;\ +ln -sf "$$(realpath "$(1)-$(3)")" "$(1)" +endef + +define gomodver +$(shell go list -m -f '{{if .Replace}}{{.Replace.Version}}{{else}}{{.Version}}{{end}}' $(1) 2>/dev/null) +endef diff --git a/PROJECT b/PROJECT new file mode 100644 index 0000000..b273a2e --- /dev/null +++ b/PROJECT @@ -0,0 +1,11 @@ +# Code generated by tool. DO NOT EDIT. +# This file is used to track the info used to scaffold your project +# and allow the plugins properly work. +# More info: https://book.kubebuilder.io/reference/project-config.html +cliVersion: 4.15.0 +domain: capi.stackhpc.com +layout: +- go.kubebuilder.io/v4 +projectName: cluster-api-janitor-openstack +repo: github.com/azimuth-cloud/cluster-api-janitor-openstack +version: "3" diff --git a/cmd/main.go b/cmd/main.go new file mode 100644 index 0000000..e2bc30f --- /dev/null +++ b/cmd/main.go @@ -0,0 +1,205 @@ +/* +Copyright 2026. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "crypto/tls" + "flag" + "os" + + // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) + // to ensure that exec-entrypoint and run can make use of them. + _ "k8s.io/client-go/plugin/pkg/client/auth" + + "k8s.io/apimachinery/pkg/runtime" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/healthz" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + "sigs.k8s.io/controller-runtime/pkg/metrics/filters" + metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" + "sigs.k8s.io/controller-runtime/pkg/webhook" + infrav1 "sigs.k8s.io/cluster-api-provider-openstack/api/v1beta1" + + "github.com/azimuth-cloud/cluster-api-janitor-openstack/internal/controller" + // +kubebuilder:scaffold:imports +) + +var ( + scheme = runtime.NewScheme() + setupLog = ctrl.Log.WithName("setup") +) + +func init() { + utilruntime.Must(clientgoscheme.AddToScheme(scheme)) + utilruntime.Must(infrav1.AddToScheme(scheme)) + // +kubebuilder:scaffold:scheme +} + +// nolint:gocyclo +func main() { + var metricsAddr string + var metricsCertPath, metricsCertName, metricsCertKey string + var webhookCertPath, webhookCertName, webhookCertKey string + var enableLeaderElection bool + var probeAddr string + var secureMetrics bool + var enableHTTP2 bool + var tlsOpts []func(*tls.Config) + flag.StringVar(&metricsAddr, "metrics-bind-address", "0", "The address the metrics endpoint binds to. "+ + "Use :8443 for HTTPS or :8080 for HTTP, or leave as 0 to disable the metrics service.") + flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.") + flag.BoolVar(&enableLeaderElection, "leader-elect", false, + "Enable leader election for controller manager. "+ + "Enabling this will ensure there is only one active controller manager.") + flag.BoolVar(&secureMetrics, "metrics-secure", true, + "If set, the metrics endpoint is served securely via HTTPS. Use --metrics-secure=false to use HTTP instead.") + flag.StringVar(&webhookCertPath, "webhook-cert-path", "", "The directory that contains the webhook certificate.") + flag.StringVar(&webhookCertName, "webhook-cert-name", "tls.crt", "The name of the webhook certificate file.") + flag.StringVar(&webhookCertKey, "webhook-cert-key", "tls.key", "The name of the webhook key file.") + flag.StringVar(&metricsCertPath, "metrics-cert-path", "", + "The directory that contains the metrics server certificate.") + flag.StringVar(&metricsCertName, "metrics-cert-name", "tls.crt", "The name of the metrics server certificate file.") + flag.StringVar(&metricsCertKey, "metrics-cert-key", "tls.key", "The name of the metrics server key file.") + flag.BoolVar(&enableHTTP2, "enable-http2", false, + "If set, HTTP/2 will be enabled for the metrics and webhook servers") + opts := zap.Options{ + Development: true, + } + opts.BindFlags(flag.CommandLine) + flag.Parse() + + ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts))) + + // if the enable-http2 flag is false (the default), http/2 should be disabled + // due to its vulnerabilities. More specifically, disabling http/2 will + // prevent from being vulnerable to the HTTP/2 Stream Cancellation and + // Rapid Reset CVEs. For more information see: + // - https://github.com/advisories/GHSA-qppj-fm5r-hxr3 + // - https://github.com/advisories/GHSA-4374-p667-p6c8 + disableHTTP2 := func(c *tls.Config) { + setupLog.Info("Disabling HTTP/2") + c.NextProtos = []string{"http/1.1"} + } + + if !enableHTTP2 { + tlsOpts = append(tlsOpts, disableHTTP2) + } + + // Initial webhook TLS options + webhookTLSOpts := tlsOpts + webhookServerOptions := webhook.Options{ + TLSOpts: webhookTLSOpts, + } + + if len(webhookCertPath) > 0 { + setupLog.Info("Initializing webhook certificate watcher using provided certificates", + "webhook-cert-path", webhookCertPath, "webhook-cert-name", webhookCertName, "webhook-cert-key", webhookCertKey) + + webhookServerOptions.CertDir = webhookCertPath + webhookServerOptions.CertName = webhookCertName + webhookServerOptions.KeyName = webhookCertKey + } + + webhookServer := webhook.NewServer(webhookServerOptions) + + // Metrics endpoint is enabled in 'config/default/kustomization.yaml'. The Metrics options configure the server. + // More info: + // - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.24.1/pkg/metrics/server + // - https://book.kubebuilder.io/reference/metrics.html + metricsServerOptions := metricsserver.Options{ + BindAddress: metricsAddr, + SecureServing: secureMetrics, + TLSOpts: tlsOpts, + } + + if secureMetrics { + // FilterProvider is used to protect the metrics endpoint with authn/authz. + // These configurations ensure that only authorized users and service accounts + // can access the metrics endpoint. The RBAC are configured in 'config/rbac/kustomization.yaml'. More info: + // https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.24.1/pkg/metrics/filters#WithAuthenticationAndAuthorization + metricsServerOptions.FilterProvider = filters.WithAuthenticationAndAuthorization + } + + // If the certificate is not specified, controller-runtime will automatically + // generate self-signed certificates for the metrics server. While convenient for development and testing, + // this setup is not recommended for production. + // + // TODO(user): If you enable certManager, uncomment the following lines: + // - [METRICS-WITH-CERTS] at config/default/kustomization.yaml to generate and use certificates + // managed by cert-manager for the metrics server. + // - [PROMETHEUS-WITH-CERTS] at config/prometheus/kustomization.yaml for TLS certification. + if len(metricsCertPath) > 0 { + setupLog.Info("Initializing metrics certificate watcher using provided certificates", + "metrics-cert-path", metricsCertPath, "metrics-cert-name", metricsCertName, "metrics-cert-key", metricsCertKey) + + metricsServerOptions.CertDir = metricsCertPath + metricsServerOptions.CertName = metricsCertName + metricsServerOptions.KeyName = metricsCertKey + } + + mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ + Scheme: scheme, + Metrics: metricsServerOptions, + WebhookServer: webhookServer, + HealthProbeBindAddress: probeAddr, + LeaderElection: enableLeaderElection, + LeaderElectionID: "4e3e11ff.capi.stackhpc.com", + // LeaderElectionReleaseOnCancel defines if the leader should step down voluntarily + // when the Manager ends. This requires the binary to immediately end when the + // Manager is stopped, otherwise, this setting is unsafe. Setting this significantly + // speeds up voluntary leader transitions as the new leader don't have to wait + // LeaseDuration time first. + // + // In the default scaffold provided, the program ends immediately after + // the manager stops, so would be fine to enable this option. However, + // if you are doing or is intended to do any operation such as perform cleanups + // after the manager stops then its usage might be unsafe. + // LeaderElectionReleaseOnCancel: true, + }) + if err != nil { + setupLog.Error(err, "Failed to start manager") + os.Exit(1) + } + + if err := (&controller.OpenStackClusterReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + DefaultVolumesPolicy: controller.DefaultVolumesFromEnv(), + RetryDefaultDelay: controller.RetryDelayFromEnv(), + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "Failed to set up controller", "controller", "OpenStackCluster") + os.Exit(1) + } + // +kubebuilder:scaffold:builder + + if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { + setupLog.Error(err, "Failed to set up health check") + os.Exit(1) + } + if err := mgr.AddReadyzCheck("readyz", healthz.Ping); err != nil { + setupLog.Error(err, "Failed to set up ready check") + os.Exit(1) + } + + setupLog.Info("Starting manager") + if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil { + setupLog.Error(err, "Failed to run manager") + os.Exit(1) + } +} diff --git a/config/default/cert_metrics_manager_patch.yaml b/config/default/cert_metrics_manager_patch.yaml new file mode 100644 index 0000000..d975015 --- /dev/null +++ b/config/default/cert_metrics_manager_patch.yaml @@ -0,0 +1,30 @@ +# This patch adds the args, volumes, and ports to allow the manager to use the metrics-server certs. + +# Add the volumeMount for the metrics-server certs +- op: add + path: /spec/template/spec/containers/0/volumeMounts/- + value: + mountPath: /tmp/k8s-metrics-server/metrics-certs + name: metrics-certs + readOnly: true + +# Add the --metrics-cert-path argument for the metrics server +- op: add + path: /spec/template/spec/containers/0/args/- + value: --metrics-cert-path=/tmp/k8s-metrics-server/metrics-certs + +# Add the metrics-server certs volume configuration +- op: add + path: /spec/template/spec/volumes/- + value: + name: metrics-certs + secret: + secretName: metrics-server-cert + optional: false + items: + - key: ca.crt + path: ca.crt + - key: tls.crt + path: tls.crt + - key: tls.key + path: tls.key diff --git a/config/default/kustomization.yaml b/config/default/kustomization.yaml new file mode 100644 index 0000000..a30e017 --- /dev/null +++ b/config/default/kustomization.yaml @@ -0,0 +1,234 @@ +# Adds namespace to all resources. +namespace: cluster-api-janitor-openstack-system + +# Value of this field is prepended to the +# names of all resources, e.g. a deployment named +# "wordpress" becomes "alices-wordpress". +# Note that it should also match with the prefix (text before '-') of the namespace +# field above. +namePrefix: cluster-api-janitor-openstack- + +# Labels to add to all resources and selectors. +#labels: +#- includeSelectors: true +# pairs: +# someName: someValue + +resources: +#- ../crd +- ../rbac +- ../manager +# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in +# crd/kustomization.yaml +#- ../webhook +# [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'. 'WEBHOOK' components are required. +#- ../certmanager +# [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'. +#- ../prometheus +# [METRICS] Expose the controller manager metrics service. +- metrics_service.yaml +# [NETWORK POLICY] Protect the /metrics endpoint and Webhook Server with NetworkPolicy. +# Only Pod(s) running a namespace labeled with 'metrics: enabled' will be able to gather the metrics. +# Only CR(s) which requires webhooks and are applied on namespaces labeled with 'webhooks: enabled' will +# be able to communicate with the Webhook Server. +#- ../network-policy + +# Uncomment the patches line if you enable Metrics +patches: +# [METRICS] The following patch will enable the metrics endpoint using HTTPS and the port :8443. +# More info: https://book.kubebuilder.io/reference/metrics +- path: manager_metrics_patch.yaml + target: + kind: Deployment + +# Uncomment the patches line if you enable Metrics and CertManager +# [METRICS-WITH-CERTS] To enable metrics protected with certManager, uncomment the following line. +# This patch will protect the metrics with certManager self-signed certs. +#- path: cert_metrics_manager_patch.yaml +# target: +# kind: Deployment + +# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in +# crd/kustomization.yaml +#- path: manager_webhook_patch.yaml +# target: +# kind: Deployment + +# [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER' prefix. +# Uncomment the following replacements to add the cert-manager CA injection annotations +#replacements: +# - source: # Uncomment the following block to enable certificates for metrics +# kind: Service +# version: v1 +# name: controller-manager-metrics-service +# fieldPath: metadata.name +# targets: +# - select: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: metrics-certs +# fieldPaths: +# - spec.dnsNames.0 +# - spec.dnsNames.1 +# options: +# delimiter: '.' +# index: 0 +# create: true +# - select: # Uncomment the following to set the Service name for TLS config in Prometheus ServiceMonitor +# kind: ServiceMonitor +# group: monitoring.coreos.com +# version: v1 +# name: controller-manager-metrics-monitor +# fieldPaths: +# - spec.endpoints.0.tlsConfig.serverName +# options: +# delimiter: '.' +# index: 0 +# create: true + +# - source: +# kind: Service +# version: v1 +# name: controller-manager-metrics-service +# fieldPath: metadata.namespace +# targets: +# - select: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: metrics-certs +# fieldPaths: +# - spec.dnsNames.0 +# - spec.dnsNames.1 +# options: +# delimiter: '.' +# index: 1 +# create: true +# - select: # Uncomment the following to set the Service namespace for TLS in Prometheus ServiceMonitor +# kind: ServiceMonitor +# group: monitoring.coreos.com +# version: v1 +# name: controller-manager-metrics-monitor +# fieldPaths: +# - spec.endpoints.0.tlsConfig.serverName +# options: +# delimiter: '.' +# index: 1 +# create: true + +# - source: # Uncomment the following block if you have any webhook +# kind: Service +# version: v1 +# name: webhook-service +# fieldPath: .metadata.name # Name of the service +# targets: +# - select: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: serving-cert +# fieldPaths: +# - .spec.dnsNames.0 +# - .spec.dnsNames.1 +# options: +# delimiter: '.' +# index: 0 +# create: true +# - source: +# kind: Service +# version: v1 +# name: webhook-service +# fieldPath: .metadata.namespace # Namespace of the service +# targets: +# - select: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: serving-cert +# fieldPaths: +# - .spec.dnsNames.0 +# - .spec.dnsNames.1 +# options: +# delimiter: '.' +# index: 1 +# create: true + +# - source: # Uncomment the following block if you have a ValidatingWebhook (--programmatic-validation) +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: serving-cert # This name should match the one in certificate.yaml +# fieldPath: .metadata.namespace # Namespace of the certificate CR +# targets: +# - select: +# kind: ValidatingWebhookConfiguration +# fieldPaths: +# - .metadata.annotations.[cert-manager.io/inject-ca-from] +# options: +# delimiter: '/' +# index: 0 +# create: true +# - source: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: serving-cert +# fieldPath: .metadata.name +# targets: +# - select: +# kind: ValidatingWebhookConfiguration +# fieldPaths: +# - .metadata.annotations.[cert-manager.io/inject-ca-from] +# options: +# delimiter: '/' +# index: 1 +# create: true + +# - source: # Uncomment the following block if you have a DefaultingWebhook (--defaulting ) +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: serving-cert +# fieldPath: .metadata.namespace # Namespace of the certificate CR +# targets: +# - select: +# kind: MutatingWebhookConfiguration +# fieldPaths: +# - .metadata.annotations.[cert-manager.io/inject-ca-from] +# options: +# delimiter: '/' +# index: 0 +# create: true +# - source: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: serving-cert +# fieldPath: .metadata.name +# targets: +# - select: +# kind: MutatingWebhookConfiguration +# fieldPaths: +# - .metadata.annotations.[cert-manager.io/inject-ca-from] +# options: +# delimiter: '/' +# index: 1 +# create: true + +# - source: # Uncomment the following block if you have a ConversionWebhook (--conversion) +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: serving-cert +# fieldPath: .metadata.namespace # Namespace of the certificate CR +# targets: # Do not remove or uncomment the following scaffold marker; required to generate code for target CRD. +# +kubebuilder:scaffold:crdkustomizecainjectionns +# - source: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: serving-cert +# fieldPath: .metadata.name +# targets: # Do not remove or uncomment the following scaffold marker; required to generate code for target CRD. +# +kubebuilder:scaffold:crdkustomizecainjectionname diff --git a/config/default/manager_metrics_patch.yaml b/config/default/manager_metrics_patch.yaml new file mode 100644 index 0000000..2aaef65 --- /dev/null +++ b/config/default/manager_metrics_patch.yaml @@ -0,0 +1,4 @@ +# This patch adds the args to allow exposing the metrics endpoint using HTTPS +- op: add + path: /spec/template/spec/containers/0/args/0 + value: --metrics-bind-address=:8443 diff --git a/config/default/metrics_service.yaml b/config/default/metrics_service.yaml new file mode 100644 index 0000000..6c012aa --- /dev/null +++ b/config/default/metrics_service.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: Service +metadata: + labels: + control-plane: controller-manager + app.kubernetes.io/name: cluster-api-janitor-openstack + app.kubernetes.io/managed-by: kustomize + name: controller-manager-metrics-service + namespace: system +spec: + ports: + - name: https + port: 8443 + protocol: TCP + targetPort: 8443 + selector: + control-plane: controller-manager + app.kubernetes.io/name: cluster-api-janitor-openstack diff --git a/config/manager/kustomization.yaml b/config/manager/kustomization.yaml new file mode 100644 index 0000000..5c5f0b8 --- /dev/null +++ b/config/manager/kustomization.yaml @@ -0,0 +1,2 @@ +resources: +- manager.yaml diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml new file mode 100644 index 0000000..fb1d4da --- /dev/null +++ b/config/manager/manager.yaml @@ -0,0 +1,102 @@ +apiVersion: v1 +kind: Namespace +metadata: + labels: + control-plane: controller-manager + app.kubernetes.io/name: cluster-api-janitor-openstack + app.kubernetes.io/managed-by: kustomize + name: system +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: controller-manager + namespace: system + labels: + control-plane: controller-manager + app.kubernetes.io/name: cluster-api-janitor-openstack + app.kubernetes.io/managed-by: kustomize +spec: + selector: + matchLabels: + control-plane: controller-manager + app.kubernetes.io/name: cluster-api-janitor-openstack + replicas: 1 + template: + metadata: + annotations: + kubectl.kubernetes.io/default-container: manager + labels: + control-plane: controller-manager + app.kubernetes.io/name: cluster-api-janitor-openstack + spec: + # TODO(user): Uncomment the following code to configure the nodeAffinity expression + # according to the platforms which are supported by your solution. + # It is considered best practice to support multiple architectures. You can + # build your manager image using the makefile target docker-buildx. + # affinity: + # nodeAffinity: + # requiredDuringSchedulingIgnoredDuringExecution: + # nodeSelectorTerms: + # - matchExpressions: + # - key: kubernetes.io/arch + # operator: In + # values: + # - amd64 + # - arm64 + # - ppc64le + # - s390x + # - key: kubernetes.io/os + # operator: In + # values: + # - linux + securityContext: + # Projects are configured by default to adhere to the "restricted" Pod Security Standards. + # This ensures that deployments meet the highest security requirements for Kubernetes. + # For more details, see: https://kubernetes.io/docs/concepts/security/pod-security-standards/#restricted + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + containers: + - command: + - /manager + args: + - --leader-elect + - --health-probe-bind-address=:8081 + image: controller:latest + name: manager + ports: + - containerPort: 8081 + name: health + protocol: TCP + securityContext: + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + capabilities: + drop: + - "ALL" + livenessProbe: + httpGet: + path: /healthz + port: 8081 + initialDelaySeconds: 15 + periodSeconds: 20 + readinessProbe: + httpGet: + path: /readyz + port: 8081 + initialDelaySeconds: 5 + periodSeconds: 10 + # TODO(user): Configure the resources accordingly based on the project requirements. + # More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + resources: + limits: + cpu: 500m + memory: 128Mi + requests: + cpu: 10m + memory: 64Mi + volumeMounts: [] + volumes: [] + serviceAccountName: controller-manager + terminationGracePeriodSeconds: 10 diff --git a/config/network-policy/allow-metrics-traffic.yaml b/config/network-policy/allow-metrics-traffic.yaml new file mode 100644 index 0000000..1f18b6b --- /dev/null +++ b/config/network-policy/allow-metrics-traffic.yaml @@ -0,0 +1,27 @@ +# This NetworkPolicy allows ingress traffic +# with Pods running on namespaces labeled with 'metrics: enabled'. Only Pods on those +# namespaces are able to gather data from the metrics endpoint. +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + labels: + app.kubernetes.io/name: cluster-api-janitor-openstack + app.kubernetes.io/managed-by: kustomize + name: allow-metrics-traffic + namespace: system +spec: + podSelector: + matchLabels: + control-plane: controller-manager + app.kubernetes.io/name: cluster-api-janitor-openstack + policyTypes: + - Ingress + ingress: + # This allows ingress traffic from any namespace with the label metrics: enabled + - from: + - namespaceSelector: + matchLabels: + metrics: enabled # Only from namespaces with this label + ports: + - port: 8443 + protocol: TCP diff --git a/config/network-policy/kustomization.yaml b/config/network-policy/kustomization.yaml new file mode 100644 index 0000000..ec0fb5e --- /dev/null +++ b/config/network-policy/kustomization.yaml @@ -0,0 +1,2 @@ +resources: +- allow-metrics-traffic.yaml diff --git a/config/prometheus/kustomization.yaml b/config/prometheus/kustomization.yaml new file mode 100644 index 0000000..fdc5481 --- /dev/null +++ b/config/prometheus/kustomization.yaml @@ -0,0 +1,11 @@ +resources: +- monitor.yaml + +# [PROMETHEUS-WITH-CERTS] The following patch configures the ServiceMonitor in ../prometheus +# to securely reference certificates created and managed by cert-manager. +# Additionally, ensure that you uncomment the [METRICS WITH CERTMANAGER] patch under config/default/kustomization.yaml +# to mount the "metrics-server-cert" secret in the Manager Deployment. +#patches: +# - path: monitor_tls_patch.yaml +# target: +# kind: ServiceMonitor diff --git a/config/prometheus/monitor.yaml b/config/prometheus/monitor.yaml new file mode 100644 index 0000000..9580214 --- /dev/null +++ b/config/prometheus/monitor.yaml @@ -0,0 +1,27 @@ +# Prometheus Monitor Service (Metrics) +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + labels: + control-plane: controller-manager + app.kubernetes.io/name: cluster-api-janitor-openstack + app.kubernetes.io/managed-by: kustomize + name: controller-manager-metrics-monitor + namespace: system +spec: + endpoints: + - path: /metrics + port: https # Ensure this is the name of the port that exposes HTTPS metrics + scheme: https + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + tlsConfig: + # TODO(user): The option insecureSkipVerify: true is not recommended for production since it disables + # certificate verification, exposing the system to potential man-in-the-middle attacks. + # For production environments, it is recommended to use cert-manager for automatic TLS certificate management. + # To apply this configuration, enable cert-manager and use the patch located at config/prometheus/servicemonitor_tls_patch.yaml, + # which securely references the certificate from the 'metrics-server-cert' secret. + insecureSkipVerify: true + selector: + matchLabels: + control-plane: controller-manager + app.kubernetes.io/name: cluster-api-janitor-openstack diff --git a/config/prometheus/monitor_tls_patch.yaml b/config/prometheus/monitor_tls_patch.yaml new file mode 100644 index 0000000..5bf84ce --- /dev/null +++ b/config/prometheus/monitor_tls_patch.yaml @@ -0,0 +1,19 @@ +# Patch for Prometheus ServiceMonitor to enable secure TLS configuration +# using certificates managed by cert-manager +- op: replace + path: /spec/endpoints/0/tlsConfig + value: + # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize + serverName: SERVICE_NAME.SERVICE_NAMESPACE.svc + insecureSkipVerify: false + ca: + secret: + name: metrics-server-cert + key: ca.crt + cert: + secret: + name: metrics-server-cert + key: tls.crt + keySecret: + name: metrics-server-cert + key: tls.key diff --git a/config/rbac/kustomization.yaml b/config/rbac/kustomization.yaml new file mode 100644 index 0000000..5619aa0 --- /dev/null +++ b/config/rbac/kustomization.yaml @@ -0,0 +1,20 @@ +resources: +# All RBAC will be applied under this service account in +# the deployment namespace. You may comment out this resource +# if your manager will use a service account that exists at +# runtime. Be sure to update RoleBinding and ClusterRoleBinding +# subjects if changing service account names. +- service_account.yaml +- role.yaml +- role_binding.yaml +- leader_election_role.yaml +- leader_election_role_binding.yaml +# The following RBAC configurations are used to protect +# the metrics endpoint with authn/authz. These configurations +# ensure that only authorized users and service accounts +# can access the metrics endpoint. Comment the following +# permissions if you want to disable this protection. +# More info: https://book.kubebuilder.io/reference/metrics.html +- metrics_auth_role.yaml +- metrics_auth_role_binding.yaml +- metrics_reader_role.yaml diff --git a/config/rbac/leader_election_role.yaml b/config/rbac/leader_election_role.yaml new file mode 100644 index 0000000..501a507 --- /dev/null +++ b/config/rbac/leader_election_role.yaml @@ -0,0 +1,40 @@ +# permissions to do leader election. +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + labels: + app.kubernetes.io/name: cluster-api-janitor-openstack + app.kubernetes.io/managed-by: kustomize + name: leader-election-role +rules: +- apiGroups: + - "" + resources: + - configmaps + verbs: + - get + - list + - watch + - create + - update + - patch + - delete +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - list + - watch + - create + - update + - patch + - delete +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch diff --git a/config/rbac/leader_election_role_binding.yaml b/config/rbac/leader_election_role_binding.yaml new file mode 100644 index 0000000..2f83d3b --- /dev/null +++ b/config/rbac/leader_election_role_binding.yaml @@ -0,0 +1,15 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + app.kubernetes.io/name: cluster-api-janitor-openstack + app.kubernetes.io/managed-by: kustomize + name: leader-election-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: leader-election-role +subjects: +- kind: ServiceAccount + name: controller-manager + namespace: system diff --git a/config/rbac/metrics_auth_role.yaml b/config/rbac/metrics_auth_role.yaml new file mode 100644 index 0000000..32d2e4e --- /dev/null +++ b/config/rbac/metrics_auth_role.yaml @@ -0,0 +1,17 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: metrics-auth-role +rules: +- apiGroups: + - authentication.k8s.io + resources: + - tokenreviews + verbs: + - create +- apiGroups: + - authorization.k8s.io + resources: + - subjectaccessreviews + verbs: + - create diff --git a/config/rbac/metrics_auth_role_binding.yaml b/config/rbac/metrics_auth_role_binding.yaml new file mode 100644 index 0000000..e775d67 --- /dev/null +++ b/config/rbac/metrics_auth_role_binding.yaml @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: metrics-auth-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: metrics-auth-role +subjects: +- kind: ServiceAccount + name: controller-manager + namespace: system diff --git a/config/rbac/metrics_reader_role.yaml b/config/rbac/metrics_reader_role.yaml new file mode 100644 index 0000000..51a75db --- /dev/null +++ b/config/rbac/metrics_reader_role.yaml @@ -0,0 +1,9 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: metrics-reader +rules: +- nonResourceURLs: + - "/metrics" + verbs: + - get diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml new file mode 100644 index 0000000..6cab0a0 --- /dev/null +++ b/config/rbac/role.yaml @@ -0,0 +1,11 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/name: cluster-api-janitor-openstack + app.kubernetes.io/managed-by: kustomize + name: manager-role +rules: +- apiGroups: [""] + resources: ["pods"] + verbs: ["get", "list", "watch"] diff --git a/config/rbac/role_binding.yaml b/config/rbac/role_binding.yaml new file mode 100644 index 0000000..f75a3e8 --- /dev/null +++ b/config/rbac/role_binding.yaml @@ -0,0 +1,15 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + app.kubernetes.io/name: cluster-api-janitor-openstack + app.kubernetes.io/managed-by: kustomize + name: manager-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: manager-role +subjects: +- kind: ServiceAccount + name: controller-manager + namespace: system diff --git a/config/rbac/service_account.yaml b/config/rbac/service_account.yaml new file mode 100644 index 0000000..aad2a02 --- /dev/null +++ b/config/rbac/service_account.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + app.kubernetes.io/name: cluster-api-janitor-openstack + app.kubernetes.io/managed-by: kustomize + name: controller-manager + namespace: system diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..1702953 --- /dev/null +++ b/go.mod @@ -0,0 +1,104 @@ +module github.com/azimuth-cloud/cluster-api-janitor-openstack + +go 1.26.0 + +require ( + github.com/go-logr/logr v1.4.3 + github.com/onsi/ginkgo/v2 v2.28.2 + github.com/onsi/gomega v1.42.0 + k8s.io/api v0.36.0 + k8s.io/apimachinery v0.36.0 + k8s.io/client-go v0.36.0 + sigs.k8s.io/cluster-api-provider-openstack v0.14.6 + sigs.k8s.io/controller-runtime v0.24.1 + sigs.k8s.io/yaml v1.6.0 +) + +require ( + cel.dev/expr v0.25.1 // indirect + github.com/Masterminds/semver/v3 v3.4.0 // indirect + github.com/antlr4-go/antlr/v4 v4.13.0 // indirect + github.com/beorn7/perks v1.0.1 // indirect + github.com/blang/semver/v4 v4.0.0 // indirect + github.com/cenkalti/backoff/v5 v5.0.3 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/emicklei/go-restful/v3 v3.13.0 // indirect + github.com/evanphx/json-patch/v5 v5.9.11 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect + github.com/fsnotify/fsnotify v1.9.0 // indirect + github.com/fxamacker/cbor/v2 v2.9.0 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/go-logr/zapr v1.3.0 // indirect + github.com/go-openapi/jsonpointer v0.21.0 // indirect + github.com/go-openapi/jsonreference v0.20.2 // indirect + github.com/go-openapi/swag v0.23.0 // indirect + github.com/go-task/slim-sprig/v3 v3.0.0 // indirect + github.com/google/cel-go v0.26.0 // indirect + github.com/google/gnostic-models v0.7.0 // indirect + github.com/google/go-cmp v0.7.0 // indirect + github.com/google/pprof v0.0.0-20260115054156-294ebfa9ad83 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/gophercloud/gophercloud/v2 v2.10.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.7 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/josharian/intern v1.0.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/mailru/easyjson v0.7.7 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + github.com/prometheus/client_golang v1.23.2 // indirect + github.com/prometheus/client_model v0.6.2 // indirect + github.com/prometheus/common v0.67.5 // indirect + github.com/prometheus/procfs v0.19.2 // indirect + github.com/spf13/cobra v1.10.2 // indirect + github.com/spf13/pflag v1.0.10 // indirect + github.com/stoewer/go-strcase v1.3.0 // indirect + github.com/x448/float16 v0.8.4 // indirect + go.opentelemetry.io/auto/sdk v1.2.1 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.65.0 // indirect + go.opentelemetry.io/otel v1.41.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.40.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.40.0 // indirect + go.opentelemetry.io/otel/metric v1.41.0 // indirect + go.opentelemetry.io/otel/sdk v1.40.0 // indirect + go.opentelemetry.io/otel/trace v1.41.0 // indirect + go.opentelemetry.io/proto/otlp v1.9.0 // indirect + go.uber.org/multierr v1.11.0 // indirect + go.uber.org/zap v1.27.1 // indirect + go.yaml.in/yaml/v2 v2.4.3 // indirect + go.yaml.in/yaml/v3 v3.0.4 // indirect + golang.org/x/exp v0.0.0-20251219203646-944ab1f22d93 // indirect + golang.org/x/mod v0.32.0 // indirect + golang.org/x/net v0.49.0 // indirect + golang.org/x/oauth2 v0.34.0 // indirect + golang.org/x/sync v0.19.0 // indirect + golang.org/x/sys v0.41.0 // indirect + golang.org/x/term v0.40.0 // indirect + golang.org/x/text v0.34.0 // indirect + golang.org/x/time v0.14.0 // indirect + golang.org/x/tools v0.41.0 // indirect + gomodules.xyz/jsonpatch/v2 v2.5.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20260128011058-8636f8732409 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260128011058-8636f8732409 // indirect + google.golang.org/grpc v1.79.3 // indirect + google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af // indirect + gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect + gopkg.in/inf.v0 v0.9.1 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect + k8s.io/apiextensions-apiserver v0.36.0 // indirect + k8s.io/apiserver v0.36.0 // indirect + k8s.io/component-base v0.36.0 // indirect + k8s.io/klog/v2 v2.140.0 // indirect + k8s.io/kube-openapi v0.0.0-20260317180543-43fb72c5454a // indirect + k8s.io/streaming v0.36.0 // indirect + k8s.io/utils v0.0.0-20260210185600-b8788abfbbc2 // indirect + sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.34.0 // indirect + sigs.k8s.io/cluster-api v1.12.8 // indirect + sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect + sigs.k8s.io/randfill v1.0.0 // indirect + sigs.k8s.io/structured-merge-diff/v6 v6.4.0 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..da655db --- /dev/null +++ b/go.sum @@ -0,0 +1,265 @@ +cel.dev/expr v0.25.1 h1:1KrZg61W6TWSxuNZ37Xy49ps13NUovb66QLprthtwi4= +cel.dev/expr v0.25.1/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4= +github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0= +github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= +github.com/antlr4-go/antlr/v4 v4.13.0 h1:lxCg3LAv+EUK6t1i0y1V6/SLeUi0eKEKdhQAlS8TVTI= +github.com/antlr4-go/antlr/v4 v4.13.0/go.mod h1:pfChB/xh/Unjila75QW7+VU4TSnWnnk9UTnmpPaOR2g= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= +github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= +github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM= +github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/emicklei/go-restful/v3 v3.13.0 h1:C4Bl2xDndpU6nJ4bc1jXd+uTmYPVUwkD6bFY/oTyCes= +github.com/emicklei/go-restful/v3 v3.13.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/evanphx/json-patch v5.7.0+incompatible h1:vgGkfT/9f8zE6tvSCe74nfpAVDQ2tG6yudJd8LBksgI= +github.com/evanphx/json-patch v5.7.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/evanphx/json-patch/v5 v5.9.11 h1:/8HVnzMq13/3x9TPvjG08wUGqBTmZBsCWzjTM0wiaDU= +github.com/evanphx/json-patch/v5 v5.9.11/go.mod h1:3j+LviiESTElxA4p3EMKAB9HXj3/XEtnUf6OZxqIQTM= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k= +github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= +github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM= +github.com/fxamacker/cbor/v2 v2.9.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= +github.com/gkampitakis/ciinfo v0.3.2 h1:JcuOPk8ZU7nZQjdUhctuhQofk7BGHuIy0c9Ez8BNhXs= +github.com/gkampitakis/ciinfo v0.3.2/go.mod h1:1NIwaOcFChN4fa/B0hEBdAb6npDlFL8Bwx4dfRLRqAo= +github.com/gkampitakis/go-diff v1.3.2 h1:Qyn0J9XJSDTgnsgHRdz9Zp24RaJeKMUHg2+PDZZdC4M= +github.com/gkampitakis/go-diff v1.3.2/go.mod h1:LLgOrpqleQe26cte8s36HTWcTmMEur6OPYerdAAS9tk= +github.com/gkampitakis/go-snaps v0.5.15 h1:amyJrvM1D33cPHwVrjo9jQxX8g/7E2wYdZ+01KS3zGE= +github.com/gkampitakis/go-snaps v0.5.15/go.mod h1:HNpx/9GoKisdhw9AFOBT1N7DBs9DiHo/hGheFGBZ+mc= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ= +github.com/go-logr/zapr v1.3.0/go.mod h1:YKepepNBd1u/oyhd/yQmtjVXmm9uML4IXUgMOwR8/Gg= +github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= +github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ= +github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY= +github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE= +github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= +github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= +github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE= +github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ= +github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= +github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= +github.com/goccy/go-yaml v1.18.0 h1:8W7wMFS12Pcas7KU+VVkaiCng+kG8QiFeFwzFb+rwuw= +github.com/goccy/go-yaml v1.18.0/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/cel-go v0.26.0 h1:DPGjXackMpJWH680oGY4lZhYjIameYmR+/6RBdDGmaI= +github.com/google/cel-go v0.26.0/go.mod h1:A9O8OU9rdvrK5MQyrqfIxo1a0u4g3sF8KB6PUIaryMM= +github.com/google/gnostic-models v0.7.0 h1:qwTtogB15McXDaNqTZdzPJRHvaVJlAl+HVQnLmJEJxo= +github.com/google/gnostic-models v0.7.0/go.mod h1:whL5G0m6dmc5cPxKc5bdKdEN3UjI7OUGxBlw57miDrQ= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= +github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/pprof v0.0.0-20260115054156-294ebfa9ad83 h1:z2ogiKUYzX5Is6zr/vP9vJGqPwcdqsWjOt+V8J7+bTc= +github.com/google/pprof v0.0.0-20260115054156-294ebfa9ad83/go.mod h1:MxpfABSjhmINe3F1It9d+8exIHFvUqtLIRCdOGNXqiI= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/gophercloud/gophercloud/v2 v2.10.0 h1:NRadC0aHNvy4iMoFXj5AFiPmut/Sj3hAPAo9B59VMGc= +github.com/gophercloud/gophercloud/v2 v2.10.0/go.mod h1:Ki/ILhYZr/5EPebrPL9Ej+tUg4lqx71/YH2JWVeU+Qk= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.7 h1:X+2YciYSxvMQK0UZ7sg45ZVabVZBeBuvMkmuI2V3Fak= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.7/go.mod h1:lW34nIZuQ8UDPdkon5fmfp2l3+ZkQ2me/+oecHYLOII= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/joshdk/go-junit v1.0.0 h1:S86cUKIdwBHWwA6xCmFlf3RTLfVXYQfvanM5Uh+K6GE= +github.com/joshdk/go-junit v1.0.0/go.mod h1:TiiV0PqkaNfFXjEiyjWM3XXrhVyCa1K4Zfga6W52ung= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= +github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/maruel/natural v1.1.1 h1:Hja7XhhmvEFhcByqDoHz9QZbkWey+COd9xWfCfn1ioo= +github.com/maruel/natural v1.1.1/go.mod h1:v+Rfd79xlw1AgVBjbO0BEQmptqb5HvL/k9GRHB7ZKEg= +github.com/mfridman/tparse v0.18.0 h1:wh6dzOKaIwkUGyKgOntDW4liXSo37qg5AXbIhkMV3vE= +github.com/mfridman/tparse v0.18.0/go.mod h1:gEvqZTuCgEhPbYk/2lS3Kcxg1GmTxxU7kTC8DvP0i/A= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee h1:W5t00kpgFdJifH4BDsTlE89Zl93FEloxaWZfGcifgq8= +github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/onsi/ginkgo/v2 v2.28.2 h1:DTrMfpqxiNUyQ3Y0zhn1n3cOO2euFgQPYIpkWwxVFps= +github.com/onsi/ginkgo/v2 v2.28.2/go.mod h1:CLtbVInNckU3/+gC8LzkGUb9oF+e8W8TdUsxPwvdOgE= +github.com/onsi/gomega v1.42.0 h1:CJby8u36xb7v34W78F8WKvqTQP7PCMIPB78IVDB73l4= +github.com/onsi/gomega v1.42.0/go.mod h1:M/Uqpu/8qTjtzCLUA2zJHX9Iilrau25x1PdoSRbWh5A= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o= +github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg= +github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= +github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= +github.com/prometheus/common v0.67.5 h1:pIgK94WWlQt1WLwAC5j2ynLaBRDiinoAb86HZHTUGI4= +github.com/prometheus/common v0.67.5/go.mod h1:SjE/0MzDEEAyrdr5Gqc6G+sXI67maCxzaT3A2+HqjUw= +github.com/prometheus/procfs v0.19.2 h1:zUMhqEW66Ex7OXIiDkll3tl9a1ZdilUOd/F6ZXw4Vws= +github.com/prometheus/procfs v0.19.2/go.mod h1:M0aotyiemPhBCM0z5w87kL22CxfcH05ZpYlu+b4J7mw= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU= +github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4= +github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk= +github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stoewer/go-strcase v1.3.0 h1:g0eASXYtp+yvN9fK8sH94oCIk0fau9uV1/ZdJ0AVEzs= +github.com/stoewer/go-strcase v1.3.0/go.mod h1:fAH5hQ5pehh+j3nZfvwdk2RgEgQjAoM8wodgtPmh1xo= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/tidwall/gjson v1.18.0 h1:FIDeeyB800efLX89e5a8Y0BNH+LOngJyGrIWxG2FKQY= +github.com/tidwall/gjson v1.18.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= +github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= +github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= +github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY= +github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28= +github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= +github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.65.0 h1:7iP2uCb7sGddAr30RRS6xjKy7AZ2JtTOPA3oolgVSw8= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.65.0/go.mod h1:c7hN3ddxs/z6q9xwvfLPk+UHlWRQyaeR1LdgfL/66l0= +go.opentelemetry.io/otel v1.41.0 h1:YlEwVsGAlCvczDILpUXpIpPSL/VPugt7zHThEMLce1c= +go.opentelemetry.io/otel v1.41.0/go.mod h1:Yt4UwgEKeT05QbLwbyHXEwhnjxNO6D8L5PQP51/46dE= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.40.0 h1:QKdN8ly8zEMrByybbQgv8cWBcdAarwmIPZ6FThrWXJs= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.40.0/go.mod h1:bTdK1nhqF76qiPoCCdyFIV+N/sRHYXYCTQc+3VCi3MI= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.40.0 h1:DvJDOPmSWQHWywQS6lKL+pb8s3gBLOZUtw4N+mavW1I= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.40.0/go.mod h1:EtekO9DEJb4/jRyN4v4Qjc2yA7AtfCBuz2FynRUWTXs= +go.opentelemetry.io/otel/metric v1.41.0 h1:rFnDcs4gRzBcsO9tS8LCpgR0dxg4aaxWlJxCno7JlTQ= +go.opentelemetry.io/otel/metric v1.41.0/go.mod h1:xPvCwd9pU0VN8tPZYzDZV/BMj9CM9vs00GuBjeKhJps= +go.opentelemetry.io/otel/sdk v1.40.0 h1:KHW/jUzgo6wsPh9At46+h4upjtccTmuZCFAc9OJ71f8= +go.opentelemetry.io/otel/sdk v1.40.0/go.mod h1:Ph7EFdYvxq72Y8Li9q8KebuYUr2KoeyHx0DRMKrYBUE= +go.opentelemetry.io/otel/sdk/metric v1.40.0 h1:mtmdVqgQkeRxHgRv4qhyJduP3fYJRMX4AtAlbuWdCYw= +go.opentelemetry.io/otel/sdk/metric v1.40.0/go.mod h1:4Z2bGMf0KSK3uRjlczMOeMhKU2rhUqdWNoKcYrtcBPg= +go.opentelemetry.io/otel/trace v1.41.0 h1:Vbk2co6bhj8L59ZJ6/xFTskY+tGAbOnCtQGVVa9TIN0= +go.opentelemetry.io/otel/trace v1.41.0/go.mod h1:U1NU4ULCoxeDKc09yCWdWe+3QoyweJcISEVa1RBzOis= +go.opentelemetry.io/proto/otlp v1.9.0 h1:l706jCMITVouPOqEnii2fIAuO3IVGBRPV5ICjceRb/A= +go.opentelemetry.io/proto/otlp v1.9.0/go.mod h1:xE+Cx5E/eEHw+ISFkwPLwCZefwVjY+pqKg1qcK03+/4= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.27.1 h1:08RqriUEv8+ArZRYSTXy1LeBScaMpVSTBhCeaZYfMYc= +go.uber.org/zap v1.27.1/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0= +go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8= +go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= +golang.org/x/exp v0.0.0-20251219203646-944ab1f22d93 h1:fQsdNF2N+/YewlRZiricy4P1iimyPKZ/xwniHj8Q2a0= +golang.org/x/exp v0.0.0-20251219203646-944ab1f22d93/go.mod h1:EPRbTFwzwjXj9NpYyyrvenVh9Y+GFeEvMNh7Xuz7xgU= +golang.org/x/mod v0.32.0 h1:9F4d3PHLljb6x//jOyokMv3eX+YDeepZSEo3mFJy93c= +golang.org/x/mod v0.32.0/go.mod h1:SgipZ/3h2Ci89DlEtEXWUk/HteuRin+HHhN+WbNhguU= +golang.org/x/net v0.49.0 h1:eeHFmOGUTtaaPSGNmjBKpbng9MulQsJURQUAfUwY++o= +golang.org/x/net v0.49.0/go.mod h1:/ysNB2EvaqvesRkuLAyjI1ycPZlQHM3q01F02UY/MV8= +golang.org/x/oauth2 v0.34.0 h1:hqK/t4AKgbqWkdkcAeI8XLmbK+4m4G5YeQRrmiotGlw= +golang.org/x/oauth2 v0.34.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA= +golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4= +golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k= +golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/term v0.40.0 h1:36e4zGLqU4yhjlmxEaagx2KuYbJq3EwY8K943ZsHcvg= +golang.org/x/term v0.40.0/go.mod h1:w2P8uVp06p2iyKKuvXIm7N/y0UCRt3UfJTfZ7oOpglM= +golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk= +golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA= +golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI= +golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4= +golang.org/x/tools v0.41.0 h1:a9b8iMweWG+S0OBnlU36rzLp20z1Rp10w+IY2czHTQc= +golang.org/x/tools v0.41.0/go.mod h1:XSY6eDqxVNiYgezAVqqCeihT4j1U2CCsqvH3WhQpnlg= +gomodules.xyz/jsonpatch/v2 v2.5.0 h1:JELs8RLM12qJGXU4u/TO3V25KW8GreMKl9pdkk14RM0= +gomodules.xyz/jsonpatch/v2 v2.5.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= +gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= +gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= +google.golang.org/genproto/googleapis/api v0.0.0-20260128011058-8636f8732409 h1:merA0rdPeUV3YIIfHHcH4qBkiQAc1nfCKSI7lB4cV2M= +google.golang.org/genproto/googleapis/api v0.0.0-20260128011058-8636f8732409/go.mod h1:fl8J1IvUjCilwZzQowmw2b7HQB2eAuYBabMXzWurF+I= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260128011058-8636f8732409 h1:H86B94AW+VfJWDqFeEbBPhEtHzJwJfTbgE2lZa54ZAQ= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260128011058-8636f8732409/go.mod h1:j9x/tPzZkyxcgEFkiKEEGxfvyumM01BEtsW8xzOahRQ= +google.golang.org/grpc v1.79.3 h1:sybAEdRIEtvcD68Gx7dmnwjZKlyfuc61Dyo9pGXXkKE= +google.golang.org/grpc v1.79.3/go.mod h1:KmT0Kjez+0dde/v2j9vzwoAScgEPx/Bw1CYChhHLrHQ= +google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af h1:+5/Sw3GsDNlEmu7TfklWKPdQ0Ykja5VEmq2i817+jbI= +google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/evanphx/json-patch.v4 v4.13.0 h1:czT3CmqEaQ1aanPc5SdlgQrrEIb8w/wwCvWWnfEbYzo= +gopkg.in/evanphx/json-patch.v4 v4.13.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +k8s.io/api v0.36.0 h1:SgqDhZzHdOtMk40xVSvCXkP9ME0H05hPM3p9AB1kL80= +k8s.io/api v0.36.0/go.mod h1:m1LVrGPNYax5NBHdO+QuAedXyuzTt4RryI/qnmNvs34= +k8s.io/apiextensions-apiserver v0.36.0 h1:Wt7E8J+VBCbj4FjiBfDTK/neXDDjyJVJc7xfuOHImZ0= +k8s.io/apiextensions-apiserver v0.36.0/go.mod h1:kGDjH0msuiIB3tgsYRV0kS9GqpMYMUsQ3GHv7TApyug= +k8s.io/apimachinery v0.36.0 h1:jZyPzhd5Z+3h9vJLt0z9XdzW9VzNzWAUw+P1xZ9PXtQ= +k8s.io/apimachinery v0.36.0/go.mod h1:FklypaRJt6n5wUIwWXIP6GJlIpUizTgfo1T/As+Tyxc= +k8s.io/apiserver v0.36.0 h1:Jg5OFAENUACByUCg15CmhZAYrr5ZyJ+jodyA1mHl3YE= +k8s.io/apiserver v0.36.0/go.mod h1:mHvwdHf+qKEm+1/hYm756SV+oREOKSPnsjagOpx6Vho= +k8s.io/client-go v0.36.0 h1:pOYi7C4RHChYjMiHpZSpSbIM6ZxVbRXBy7CuiIwqA3c= +k8s.io/client-go v0.36.0/go.mod h1:ZKKcpwF0aLYfkHFCjillCKaTK/yBkEDHTDXCFY6AS9Y= +k8s.io/component-base v0.36.0 h1:hFjEktssxiJhrK1zfybkH4kJOi8iZuF+mIDCqS5+jRo= +k8s.io/component-base v0.36.0/go.mod h1:JZvIfcNHk+uck+8LhJzhSBtydWXaZNQwX2OdL+Mnwsk= +k8s.io/klog/v2 v2.140.0 h1:Tf+J3AH7xnUzZyVVXhTgGhEKnFqye14aadWv7bzXdzc= +k8s.io/klog/v2 v2.140.0/go.mod h1:o+/RWfJ6PwpnFn7OyAG3QnO47BFsymfEfrz6XyYSSp0= +k8s.io/kube-openapi v0.0.0-20260317180543-43fb72c5454a h1:xCeOEAOoGYl2jnJoHkC3hkbPJgdATINPMAxaynU2Ovg= +k8s.io/kube-openapi v0.0.0-20260317180543-43fb72c5454a/go.mod h1:uGBT7iTA6c6MvqUvSXIaYZo9ukscABYi2btjhvgKGZ0= +k8s.io/streaming v0.36.0 h1:agnTxU+NFulUrtYzXUGKO3ndEa8jKwht1Kwn9nu9x+4= +k8s.io/streaming v0.36.0/go.mod h1:z6fV3D+NVkoeqRMtWwlUZK6U17SY/LqNzOxWL6GyR/s= +k8s.io/utils v0.0.0-20260210185600-b8788abfbbc2 h1:AZYQSJemyQB5eRxqcPky+/7EdBj0xi3g0ZcxxJ7vbWU= +k8s.io/utils v0.0.0-20260210185600-b8788abfbbc2/go.mod h1:xDxuJ0whA3d0I4mf/C4ppKHxXynQ+fxnkmQH0vTHnuk= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.34.0 h1:hSfpvjjTQXQY2Fol2CS0QHMNs/WI1MOSGzCm1KhM5ec= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.34.0/go.mod h1:Ve9uj1L+deCXFrPOk1LpFXqTg7LCFzFso6PA48q/XZw= +sigs.k8s.io/cluster-api v1.12.8 h1:37SLcQRG9EMhmsZJwyEx8pNBkmY1Xhog53slhDy44m4= +sigs.k8s.io/cluster-api v1.12.8/go.mod h1:Xz6YnayDc2+/3OA1i6wlXrhmErV1KKJmyfs7JzMDn+k= +sigs.k8s.io/cluster-api-provider-openstack v0.14.6 h1:yOln3Hd6JOFe2mkOGckFdWwFNhJdcNWrlfgwMdxmzto= +sigs.k8s.io/cluster-api-provider-openstack v0.14.6/go.mod h1:A5d/QLwDgQ9vOAD6/z6EKN23+4GXZK4E8vmh7Jlfsm0= +sigs.k8s.io/controller-runtime v0.24.1 h1:miPEwrmirImAvgME1L9qebGHrOnGJoVmVdtOU9fRfo4= +sigs.k8s.io/controller-runtime v0.24.1/go.mod h1:vFkfY5fGt5xAC/sKb8IBFKgWPNKG9OUG29dR8Y2wImw= +sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 h1:IpInykpT6ceI+QxKBbEflcR5EXP7sU1kvOlxwZh5txg= +sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg= +sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU= +sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= +sigs.k8s.io/structured-merge-diff/v6 v6.4.0 h1:qmp2e3ZfFi1/jJbDGpD4mt3wyp6PE1NfKHCYLqgNQJo= +sigs.k8s.io/structured-merge-diff/v6 v6.4.0/go.mod h1:M3W8sfWvn2HhQDIbGWj3S099YozAsymCo/wrT5ohRUE= +sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs= +sigs.k8s.io/yaml v1.6.0/go.mod h1:796bPqUfzR/0jLAl6XjHl3Ck7MiyVv8dbTdyT3/pMf4= diff --git a/hack/boilerplate.go.txt b/hack/boilerplate.go.txt new file mode 100644 index 0000000..af737e6 --- /dev/null +++ b/hack/boilerplate.go.txt @@ -0,0 +1,15 @@ +/* +Copyright YEAR. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ \ No newline at end of file diff --git a/internal/controller/openstackcluster_controller.go b/internal/controller/openstackcluster_controller.go new file mode 100644 index 0000000..93995ce --- /dev/null +++ b/internal/controller/openstackcluster_controller.go @@ -0,0 +1,276 @@ +/* +Copyright 2026. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "context" + "encoding/base64" + "fmt" + "math/rand" + "os" + "strconv" + "time" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/log" + infrav1 "sigs.k8s.io/cluster-api-provider-openstack/api/v1beta1" + + "github.com/azimuth-cloud/cluster-api-janitor-openstack/internal/openstack" +) + +const ( + Finalizer = "janitor.capi.stackhpc.com" + + VolumesPolicyAnnotation = "janitor.capi.stackhpc.com/volumes-policy" + CredentialPolicyAnnotation = "janitor.capi.stackhpc.com/credential-policy" + RetryAnnotation = "janitor.capi.stackhpc.com/retry" + ClusterNameLabel = "cluster.x-k8s.io/cluster-name" + + PolicyDelete = "delete" + + defaultRetryDelay = 60 // seconds +) + +// OpenStackClusterReconciler reconciles OpenStackCluster objects from CAPO. +type OpenStackClusterReconciler struct { + client.Client + Scheme *runtime.Scheme + DefaultVolumesPolicy string + RetryDefaultDelay int +} + +//+kubebuilder:rbac:groups=infrastructure.cluster.x-k8s.io,resources=openstackclusters,verbs=get;list;watch;patch;update +//+kubebuilder:rbac:groups="",resources=secrets,verbs=get;delete +//+kubebuilder:rbac:groups="",resources=namespaces,verbs=list;watch +//+kubebuilder:rbac:groups="",resources=events,verbs=create +//+kubebuilder:rbac:groups=apiextensions.k8s.io,resources=customresourcedefinitions,verbs=get;list;watch + +func (r *OpenStackClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + logger := log.FromContext(ctx) + + var cluster infrav1.OpenStackCluster + if err := r.Get(ctx, req.NamespacedName, &cluster); err != nil { + return ctrl.Result{}, client.IgnoreNotFound(err) + } + + clusterName := clusterNameFor(&cluster) + logger = logger.WithValues("clusterName", clusterName) + logger.V(1).Info("reconciling OpenStackCluster") + + // Not deleting: ensure our finalizer is present. + if cluster.DeletionTimestamp.IsZero() { + if !controllerutil.ContainsFinalizer(&cluster, Finalizer) { + controllerutil.AddFinalizer(&cluster, Finalizer) + if err := r.Update(ctx, &cluster); err != nil { + return ctrl.Result{}, fmt.Errorf("adding finalizer: %w", err) + } + logger.Info("added janitor finalizer to cluster") + } + return ctrl.Result{}, nil + } + + // Deleting: only act if our finalizer is present. + if !controllerutil.ContainsFinalizer(&cluster, Finalizer) { + logger.Info("janitor finalizer not present, skipping cleanup") + return ctrl.Result{}, nil + } + + // Fetch the cloud credential secret. + secret, err := r.getSecret(ctx, cluster.Spec.IdentityRef.Name, req.Namespace) + if err != nil { + return ctrl.Result{}, fmt.Errorf("fetching identity secret: %w", err) + } + if secret == nil { + logger.Error(nil, "clouds.yaml secret not found", "secretName", cluster.Spec.IdentityRef.Name) + return ctrl.Result{}, nil + } + + cloudsYAML := decodeSecretField(secret.Data["clouds.yaml"]) + cacert := decodeSecretField(secret.Data["cacert"]) + + cloudName := cluster.Spec.IdentityRef.CloudName + if cloudName == "" { + cloudName = "openstack" + } + + includeVolumes := r.volumesPolicyFor(&cluster) == PolicyDelete + + credentialPolicy := secret.Annotations[CredentialPolicyAnnotation] + includeAppcred := credentialPolicy == PolicyDelete && len(cluster.Finalizers) == 1 + + purgeErr := openstack.PurgeResources(ctx, openstack.PurgeOptions{ + CloudsYAML: cloudsYAML, + CloudName: cloudName, + CACert: cacert, + ClusterName: clusterName, + IncludeVolumes: includeVolumes, + IncludeAppcred: includeAppcred, + Logger: logger, + }) + if purgeErr != nil { + logger.Error(purgeErr, "purge failed, will retry") + delay := r.retryDelay() + time.Sleep(delay) + if err := r.annotateRetry(ctx, &cluster); err != nil && !apierrors.IsNotFound(err) { + return ctrl.Result{}, err + } + return ctrl.Result{}, nil + } + + // Delete appcred secret if this is the last finalizer and policy says so. + if credentialPolicy == PolicyDelete { + if len(cluster.Finalizers) == 1 { + if err := r.deleteSecret(ctx, secret.Name, req.Namespace); err != nil && !apierrors.IsNotFound(err) { + return ctrl.Result{}, fmt.Errorf("deleting credential secret: %w", err) + } + logger.Info("cloud credential secret deleted") + } else { + // Other finalizers still present; trigger a retry when they are removed. + other := otherFinalizer(cluster.Finalizers, Finalizer) + logger.Info("waiting for other finalizer before deleting appcred", "otherFinalizer", other) + time.Sleep(5 * time.Second) + if err := r.annotateRetry(ctx, &cluster); err != nil && !apierrors.IsNotFound(err) { + return ctrl.Result{}, err + } + return ctrl.Result{}, nil + } + } + + // Remove our finalizer. + controllerutil.RemoveFinalizer(&cluster, Finalizer) + if err := r.Update(ctx, &cluster); err != nil { + return ctrl.Result{}, fmt.Errorf("removing finalizer: %w", err) + } + logger.Info("removed janitor finalizer from cluster") + return ctrl.Result{}, nil +} + +// SetupWithManager registers the reconciler with the controller manager. +func (r *OpenStackClusterReconciler) SetupWithManager(mgr ctrl.Manager) error { + return ctrl.NewControllerManagedBy(mgr). + For(&infrav1.OpenStackCluster{}). + Complete(r) +} + +// clusterNameFor returns the cluster name to use for resource cleanup. +// It prefers the cluster.x-k8s.io/cluster-name label over metadata.name. +func clusterNameFor(cluster *infrav1.OpenStackCluster) string { + if name, ok := cluster.Labels[ClusterNameLabel]; ok { + return name + } + return cluster.Name +} + +func (r *OpenStackClusterReconciler) volumesPolicyFor(cluster *infrav1.OpenStackCluster) string { + if ann, ok := cluster.Annotations[VolumesPolicyAnnotation]; ok { + return ann + } + if r.DefaultVolumesPolicy != "" { + return r.DefaultVolumesPolicy + } + return PolicyDelete +} + +func (r *OpenStackClusterReconciler) retryDelay() time.Duration { + d := r.RetryDefaultDelay + if d <= 0 { + d = defaultRetryDelay + } + return time.Duration(d) * time.Second +} + +func (r *OpenStackClusterReconciler) annotateRetry(ctx context.Context, cluster *infrav1.OpenStackCluster) error { + patch := client.MergeFrom(cluster.DeepCopy()) + if cluster.Annotations == nil { + cluster.Annotations = make(map[string]string) + } + cluster.Annotations[RetryAnnotation] = randString(8) + return r.Patch(ctx, cluster, patch) +} + +func (r *OpenStackClusterReconciler) getSecret(ctx context.Context, name, namespace string) (*corev1.Secret, error) { + var secret corev1.Secret + if err := r.Get(ctx, types.NamespacedName{Name: name, Namespace: namespace}, &secret); err != nil { + if apierrors.IsNotFound(err) { + return nil, nil + } + return nil, err + } + return &secret, nil +} + +func (r *OpenStackClusterReconciler) deleteSecret(ctx context.Context, name, namespace string) error { + var secret corev1.Secret + secret.Name = name + secret.Namespace = namespace + return r.Delete(ctx, &secret) +} + +// decodeSecretField base64-decodes a secret data field if needed. +// Kubernetes stores secret data already base64-decoded in the Go API. +func decodeSecretField(data []byte) string { + if len(data) == 0 { + return "" + } + decoded, err := base64.StdEncoding.DecodeString(string(data)) + if err != nil { + return string(data) // already raw bytes from Kubernetes API + } + return string(decoded) +} + +func randString(n int) string { + const letters = "abcdefghijklmnopqrstuvwxyz0123456789" + b := make([]byte, n) + for i := range b { + b[i] = letters[rand.Intn(len(letters))] + } + return string(b) +} + +func otherFinalizer(finalizers []string, skip string) string { + for _, f := range finalizers { + if f != skip { + return f + } + } + return "" +} + +// DefaultVolumesFromEnv reads CAPI_JANITOR_DEFAULT_VOLUMES_POLICY from environment. +func DefaultVolumesFromEnv() string { + if v := os.Getenv("CAPI_JANITOR_DEFAULT_VOLUMES_POLICY"); v != "" { + return v + } + return PolicyDelete +} + +// RetryDelayFromEnv reads CAPI_JANITOR_RETRY_DEFAULT_DELAY from environment. +func RetryDelayFromEnv() int { + if v := os.Getenv("CAPI_JANITOR_RETRY_DEFAULT_DELAY"); v != "" { + if n, err := strconv.Atoi(v); err == nil { + return n + } + } + return defaultRetryDelay +} diff --git a/internal/openstack/cloud.go b/internal/openstack/cloud.go new file mode 100644 index 0000000..e08bd87 --- /dev/null +++ b/internal/openstack/cloud.go @@ -0,0 +1,317 @@ +// Package openstack provides OpenStack client utilities and resource cleanup +// logic for the cluster-api-janitor-openstack operator. +package openstack + +import ( + "context" + "crypto/tls" + "encoding/json" + "fmt" + "io" + "net/http" + "regexp" + "strings" + + "sigs.k8s.io/yaml" +) + +const ( + // KeepProperty is the OpenStack volume metadata key that marks a volume as user-kept. + KeepProperty = "janitor.capi.azimuth-cloud.com/keep" +) + +// AuthenticationError is returned when OpenStack authentication fails. +type AuthenticationError struct { + UserID string +} + +func (e *AuthenticationError) Error() string { + return fmt.Sprintf("failed to authenticate as user: %s", e.UserID) +} + +// UnsupportedAuthTypeError is returned when clouds.yaml uses an unsupported auth type. +type UnsupportedAuthTypeError struct { + AuthType string +} + +func (e *UnsupportedAuthTypeError) Error() string { + return fmt.Sprintf("unsupported authentication type: %s", e.AuthType) +} + +// CatalogError is returned when a required service is absent from the OpenStack catalog. +type CatalogError struct { + ServiceType string +} + +func (e *CatalogError) Error() string { + return fmt.Sprintf("service type %s not found in OpenStack service catalog", e.ServiceType) +} + +// cloudsFile represents a minimal clouds.yaml structure. +type cloudsFile struct { + Clouds map[string]cloudEntry `yaml:"clouds" json:"clouds"` +} + +type cloudEntry struct { + AuthType string `yaml:"auth_type" json:"auth_type"` + Auth authBlock `yaml:"auth" json:"auth"` + Region string `yaml:"region_name" json:"region_name"` + Interface string `yaml:"interface" json:"interface"` +} + +type authBlock struct { + AuthURL string `yaml:"auth_url" json:"auth_url"` + ApplicationCredentialID string `yaml:"application_credential_id" json:"application_credential_id"` + ApplicationCredentialSecret string `yaml:"application_credential_secret" json:"application_credential_secret"` +} + +// parseCloudsYAML parses a clouds.yaml string into a cloudsFile. +func parseCloudsYAML(data string) (*cloudsFile, error) { + var cf cloudsFile + if err := yaml.Unmarshal([]byte(data), &cf); err != nil { + return nil, fmt.Errorf("parsing clouds.yaml: %w", err) + } + return &cf, nil +} + +// authURLBase strips any trailing /v3 path segment. +var v3Suffix = regexp.MustCompile(`/v3/?$`) + +func authURLBase(raw string) string { + return v3Suffix.ReplaceAllString(raw, "") +} + +// Session holds an authenticated OpenStack session with discovered endpoints. +type Session struct { + token string + userID string + endpoints map[string]string + httpClient *http.Client + authenticated bool +} + +// IsAuthenticated reports whether the session has a valid token and catalog. +func (s *Session) IsAuthenticated() bool { return s.authenticated } + +// UserID returns the ID of the authenticated user. +func (s *Session) UserID() string { return s.userID } + +// Authenticate performs token authentication against Keystone and discovers +// the service catalog, returning a ready-to-use Session. +func Authenticate(ctx context.Context, cloudsYAML, cloudName, cacert string) (*Session, error) { + cf, err := parseCloudsYAML(cloudsYAML) + if err != nil { + return nil, err + } + entry, ok := cf.Clouds[cloudName] + if !ok { + return nil, fmt.Errorf("cloud %q not found in clouds.yaml", cloudName) + } + if entry.AuthType != "v3applicationcredential" { + return nil, &UnsupportedAuthTypeError{AuthType: entry.AuthType} + } + + tlsCfg := &tls.Config{} //nolint:gosec + if cacert != "" { + if err := loadCACert(tlsCfg, cacert); err != nil { + return nil, err + } + } + hc := &http.Client{Transport: &http.Transport{TLSClientConfig: tlsCfg}} + + iface := entry.Interface + if iface == "" { + iface = "public" + } + baseURL := authURLBase(entry.Auth.AuthURL) + + s := &Session{httpClient: hc} + if err := s.getToken(ctx, baseURL, entry.Auth); err != nil { + if isHTTP(err, http.StatusNotFound) { + return s, nil // deleted appcred case: unauthenticated, no fatal error + } + return nil, err + } + if err := s.loadCatalog(ctx, baseURL, iface, entry.Region); err != nil { + return nil, err + } + return s, nil +} + +func (s *Session) getToken(ctx context.Context, baseURL string, auth authBlock) error { + body, _ := json.Marshal(map[string]any{ + "auth": map[string]any{ + "identity": map[string]any{ + "methods": []string{"application_credential"}, + "application_credential": map[string]any{ + "id": auth.ApplicationCredentialID, + "secret": auth.ApplicationCredentialSecret, + }, + }, + }, + }) + req, err := http.NewRequestWithContext(ctx, http.MethodPost, + baseURL+"/v3/auth/tokens", strings.NewReader(string(body))) + if err != nil { + return err + } + req.Header.Set("Content-Type", "application/json") + resp, err := s.httpClient.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + if resp.StatusCode >= 400 { + return &httpStatusError{code: resp.StatusCode} + } + s.token = resp.Header.Get("X-Subject-Token") + var result struct { + Token struct { + User struct { + ID string `json:"id"` + } `json:"user"` + } `json:"token"` + } + return json.NewDecoder(resp.Body).Decode(&result) +} + +func (s *Session) loadCatalog(ctx context.Context, baseURL, iface, region string) error { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, + baseURL+"/v3/auth/catalog", nil) + if err != nil { + return err + } + req.Header.Set("X-Auth-Token", s.token) + resp, err := s.httpClient.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + if resp.StatusCode == http.StatusNotFound { + return nil + } + if resp.StatusCode >= 400 { + return &httpStatusError{code: resp.StatusCode} + } + var catalog struct { + Catalog []struct { + Type string `json:"type"` + Endpoints []struct { + Interface string `json:"interface"` + RegionID string `json:"region_id"` + URL string `json:"url"` + } `json:"endpoints"` + } `json:"catalog"` + } + if err := json.NewDecoder(resp.Body).Decode(&catalog); err != nil { + return err + } + s.endpoints = make(map[string]string) + for _, entry := range catalog.Catalog { + for _, ep := range entry.Endpoints { + if ep.Interface != iface { + continue + } + if region != "" && ep.RegionID != region { + continue + } + s.endpoints[entry.Type] = ep.URL + break + } + } + if len(s.endpoints) > 0 { + s.authenticated = true + } + return nil +} + +func (s *Session) endpointFor(serviceTypes ...string) (string, error) { + for _, st := range serviceTypes { + if u, ok := s.endpoints[st]; ok { + return u, nil + } + } + return "", &CatalogError{ServiceType: strings.Join(serviceTypes, " or ")} +} + +// doGet issues an authenticated GET and returns the response body. +func (s *Session) doGet(ctx context.Context, url string) ([]byte, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return nil, err + } + req.Header.Set("X-Auth-Token", s.token) + resp, err := s.httpClient.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + if resp.StatusCode >= 400 { + return nil, &httpStatusError{code: resp.StatusCode} + } + return io.ReadAll(resp.Body) +} + +// doDelete issues an authenticated DELETE request. +func (s *Session) doDelete(ctx context.Context, url string) error { + req, err := http.NewRequestWithContext(ctx, http.MethodDelete, url, nil) + if err != nil { + return err + } + req.Header.Set("X-Auth-Token", s.token) + resp, err := s.httpClient.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + if resp.StatusCode == http.StatusNotFound { + return nil + } + if resp.StatusCode >= 400 { + return &httpStatusError{code: resp.StatusCode} + } + return nil +} + +type httpStatusError struct{ code int } + +func (e *httpStatusError) Error() string { return fmt.Sprintf("HTTP %d", e.code) } +func (e *httpStatusError) StatusCode() int { return e.code } + +func isHTTP(err error, code int) bool { + var he *httpStatusError + if errors_as(err, &he) { + return he.code == code + } + return false +} + +func errors_as(err error, target **httpStatusError) bool { + if he, ok := err.(*httpStatusError); ok { + *target = he + return true + } + return false +} + +// isTransient returns true for HTTP 400 and 409 (conflict/bad-request = retry). +func isTransient(err error) bool { + var he *httpStatusError + if errors_as(err, &he) { + return he.code == http.StatusBadRequest || he.code == http.StatusConflict + } + return false +} + +// AppCredentialID extracts the application_credential_id from a clouds.yaml string. +func AppCredentialID(cloudsYAML, cloudName string) (string, error) { + cf, err := parseCloudsYAML(cloudsYAML) + if err != nil { + return "", err + } + entry, ok := cf.Clouds[cloudName] + if !ok { + return "", fmt.Errorf("cloud %q not found", cloudName) + } + return entry.Auth.ApplicationCredentialID, nil +} diff --git a/internal/openstack/purge.go b/internal/openstack/purge.go new file mode 100644 index 0000000..35c4449 --- /dev/null +++ b/internal/openstack/purge.go @@ -0,0 +1,68 @@ +package openstack + +import ( + "context" + + "github.com/go-logr/logr" +) + +// PurgeOptions holds parameters for cleaning up OpenStack resources +// associated with a deleted Cluster API cluster. +type PurgeOptions struct { + // CloudsYAML is the decoded content of the clouds.yaml credential file. + CloudsYAML string + // CloudName is the entry name within clouds.yaml to use. + CloudName string + // CACert is an optional PEM-encoded CA certificate for TLS verification. + CACert string + // ClusterName is the CAPI cluster name used to identify owned resources. + ClusterName string + // IncludeVolumes controls whether Cinder volumes and snapshots are deleted. + IncludeVolumes bool + // IncludeAppcred controls whether the OpenStack application credential is deleted. + IncludeAppcred bool + // Logger receives structured log messages during cleanup. + Logger logr.Logger +} + +// PurgeResources removes all OpenStack resources (FIPs, load balancers, +// security groups, volumes, snapshots, and optionally the application +// credential) created by OCCM/CSI for the given cluster. +func PurgeResources(ctx context.Context, opts PurgeOptions) error { + session, err := Authenticate(ctx, opts.CloudsYAML, opts.CloudName, opts.CACert) + if err != nil { + return err + } + + if !session.IsAuthenticated() { + if opts.IncludeAppcred { + opts.Logger.Info("application credential has been deleted, skipping cleanup") + return nil + } + return &AuthenticationError{UserID: session.UserID()} + } + + if err := session.DeleteFloatingIPs(ctx, opts.Logger, opts.ClusterName); err != nil { + return err + } + if err := session.DeleteLoadBalancers(ctx, opts.Logger, opts.ClusterName); err != nil { + return err + } + if err := session.DeleteSecurityGroups(ctx, opts.Logger, opts.ClusterName); err != nil { + return err + } + if opts.IncludeVolumes { + if err := session.DeleteSnapshots(ctx, opts.Logger, opts.ClusterName); err != nil { + return err + } + if err := session.DeleteVolumes(ctx, opts.Logger, opts.ClusterName); err != nil { + return err + } + } + if opts.IncludeAppcred { + if err := session.DeleteAppCredential(ctx, opts.Logger, opts.CloudsYAML, opts.CloudName); err != nil { + return err + } + } + return nil +} diff --git a/internal/openstack/resources.go b/internal/openstack/resources.go new file mode 100644 index 0000000..be3916f --- /dev/null +++ b/internal/openstack/resources.go @@ -0,0 +1,341 @@ +package openstack + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/url" + "strings" + + "github.com/go-logr/logr" +) + +// DeleteFloatingIPs removes FIPs whose description matches the cluster. +// Expected: "Floating IP for Kubernetes external service from cluster " +func (s *Session) DeleteFloatingIPs(ctx context.Context, logger logr.Logger, cluster string) error { + networkURL, err := s.endpointFor("network") + if err != nil { + return err + } + prefix := "Floating IP for Kubernetes external service" + suffix := "from cluster " + cluster + + type fip struct { + ID string `json:"id"` + Description string `json:"description"` + } + list, err := listPages[fip](ctx, s, networkURL+"/v2.0/floatingips", "floatingips") + if err != nil { + return err + } + deleted := false + for _, f := range list { + if strings.HasPrefix(f.Description, prefix) && strings.HasSuffix(f.Description, suffix) { + logger.Info("deleting floating IP", "id", f.ID) + if err := s.doDelete(ctx, networkURL+"/v2.0/floatingips/"+f.ID); err != nil && !isTransient(err) { + return err + } + deleted = true + } + } + if deleted { + remaining, err := listPages[fip](ctx, s, networkURL+"/v2.0/floatingips", "floatingips") + if err != nil { + return err + } + for _, f := range remaining { + if strings.HasPrefix(f.Description, prefix) && strings.HasSuffix(f.Description, suffix) { + return fmt.Errorf("floating IPs still present for cluster %s", cluster) + } + } + } + logger.Info("deleted floating IPs for LoadBalancer services") + return nil +} + +// DeleteLoadBalancers removes Octavia LBs whose name starts with kube_service__ +// or follows the Azimuth naming convention. +func (s *Session) DeleteLoadBalancers(ctx context.Context, logger logr.Logger, cluster string) error { + lbURL, err := s.endpointFor("load-balancer") + if err != nil { + logger.Info("load-balancer service not found in catalog, skipping LB cleanup") + return nil + } + + type lb struct { + ID string `json:"id"` + Name string `json:"name"` + } + list, err := listPages[lb](ctx, s, lbURL+"/v2/lbaas/loadbalancers", "loadbalancers") + if err != nil { + logger.Error(err, "failed to list load balancers, some may remain") + return nil + } + kubePrefix := "kube_service_" + cluster + "_" + deleted := false + for _, l := range list { + if strings.HasPrefix(l.Name, kubePrefix) { + logger.Info("deleting load balancer", "id", l.ID, "name", l.Name) + target := lbURL + "/v2/lbaas/loadbalancers/" + l.ID + "?cascade=true" + if err := s.doDelete(ctx, target); err != nil && !isTransient(err) { + return err + } + deleted = true + } + } + if deleted { + remaining, err := listPages[lb](ctx, s, lbURL+"/v2/lbaas/loadbalancers", "loadbalancers") + if err != nil { + logger.Error(err, "failed to verify LB deletion") + return nil + } + for _, l := range remaining { + if strings.HasPrefix(l.Name, kubePrefix) { + return fmt.Errorf("load balancers still present for cluster %s", cluster) + } + } + } + logger.Info("deleted load balancers for LoadBalancer services") + return nil +} + +// DeleteSecurityGroups removes SGs whose description matches the cluster LB pattern. +// Expected: "Security Group for Service LoadBalancer in cluster " +func (s *Session) DeleteSecurityGroups(ctx context.Context, logger logr.Logger, cluster string) error { + networkURL, err := s.endpointFor("network") + if err != nil { + return err + } + sgSuffix := "Service LoadBalancer in cluster " + cluster + + type sg struct { + ID string `json:"id"` + Description string `json:"description"` + } + list, err := listPages[sg](ctx, s, networkURL+"/v2.0/security-groups", "security_groups") + if err != nil { + return err + } + deleted := false + for _, g := range list { + if strings.HasPrefix(g.Description, "Security Group for") && strings.HasSuffix(g.Description, sgSuffix) { + logger.Info("deleting security group", "id", g.ID) + if err := s.doDelete(ctx, networkURL+"/v2.0/security-groups/"+g.ID); err != nil && !isTransient(err) { + return err + } + deleted = true + } + } + if deleted { + remaining, err := listPages[sg](ctx, s, networkURL+"/v2.0/security-groups", "security_groups") + if err != nil { + return err + } + for _, g := range remaining { + if strings.HasPrefix(g.Description, "Security Group for") && strings.HasSuffix(g.Description, sgSuffix) { + return fmt.Errorf("security groups still present for cluster %s", cluster) + } + } + } + logger.Info("deleted security groups for LoadBalancer services") + return nil +} + +// volumeItem represents a Cinder volume or snapshot with its metadata. +type volumeItem struct { + ID string `json:"id"` + Metadata map[string]string `json:"metadata"` +} + +// DeleteSnapshots removes Cinder snapshots tagged with the cluster CSI metadata. +func (s *Session) DeleteSnapshots(ctx context.Context, logger logr.Logger, cluster string) error { + cinderURL, err := s.cinderEndpoint() + if err != nil { + return err + } + list, err := s.listVolumeItems(ctx, cinderURL+"/snapshots/detail", "snapshots") + if err != nil { + return err + } + deleted := false + for _, snap := range list { + if snap.Metadata["cinder.csi.openstack.org/cluster"] == cluster { + logger.Info("deleting snapshot", "id", snap.ID) + if err := s.doDelete(ctx, cinderURL+"/snapshots/"+snap.ID); err != nil && !isTransient(err) { + return err + } + deleted = true + } + } + if deleted { + remaining, err := s.listVolumeItems(ctx, cinderURL+"/snapshots/detail", "snapshots") + if err != nil { + return err + } + for _, snap := range remaining { + if snap.Metadata["cinder.csi.openstack.org/cluster"] == cluster { + return fmt.Errorf("snapshots still present for cluster %s", cluster) + } + } + } + logger.Info("deleted snapshots for persistent volume claims") + return nil +} + +// DeleteVolumes removes Cinder volumes tagged with the cluster CSI metadata, +// unless the user has set the keep property to "true". +func (s *Session) DeleteVolumes(ctx context.Context, logger logr.Logger, cluster string) error { + cinderURL, err := s.cinderEndpoint() + if err != nil { + return err + } + list, err := s.listVolumeItems(ctx, cinderURL+"/volumes/detail", "volumes") + if err != nil { + return err + } + deleted := false + for _, vol := range list { + if vol.Metadata["cinder.csi.openstack.org/cluster"] != cluster { + continue + } + if vol.Metadata[KeepProperty] == "true" { + continue + } + logger.Info("deleting volume", "id", vol.ID) + if err := s.doDelete(ctx, cinderURL+"/volumes/"+vol.ID); err != nil && !isTransient(err) { + return err + } + deleted = true + } + if deleted { + remaining, err := s.listVolumeItems(ctx, cinderURL+"/volumes/detail", "volumes") + if err != nil { + return err + } + for _, vol := range remaining { + if vol.Metadata["cinder.csi.openstack.org/cluster"] == cluster && + vol.Metadata[KeepProperty] != "true" { + return fmt.Errorf("volumes still present for cluster %s", cluster) + } + } + } + logger.Info("deleted volumes for persistent volume claims") + return nil +} + +// DeleteAppCredential removes the OpenStack application credential for the cluster. +func (s *Session) DeleteAppCredential(ctx context.Context, logger logr.Logger, cloudsYAML, cloudName string) error { + identityURL, err := s.endpointFor("identity") + if err != nil { + return err + } + appcredID, err := AppCredentialID(cloudsYAML, cloudName) + if err != nil { + return err + } + target := strings.TrimRight(identityURL, "/") + "/v3/users/" + s.userID + "/application_credentials/" + appcredID + req, err := newDeleteRequest(ctx, target, s.token) + if err != nil { + return err + } + resp, err := s.httpClient.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + switch resp.StatusCode { + case http.StatusNoContent, http.StatusOK, http.StatusNotFound: + logger.Info("deleted application credential for cluster") + return nil + case http.StatusForbidden: + logger.Info("unable to delete application credential (restricted), skipping") + return nil + default: + return fmt.Errorf("deleting application credential: HTTP %d", resp.StatusCode) + } +} + +func (s *Session) cinderEndpoint() (string, error) { + return s.endpointFor("volumev3", "block-storage") +} + +func (s *Session) listVolumeItems(ctx context.Context, endpoint, key string) ([]volumeItem, error) { + body, err := s.doGet(ctx, endpoint) + if err != nil { + return nil, err + } + var raw map[string]json.RawMessage + if err := json.Unmarshal(body, &raw); err != nil { + return nil, err + } + var items []volumeItem + if err := json.Unmarshal(raw[key], &items); err != nil { + return nil, err + } + return items, nil +} + +func newDeleteRequest(ctx context.Context, target, token string) (*http.Request, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodDelete, target, nil) + if err != nil { + return nil, err + } + req.Header.Set("X-Auth-Token", token) + return req, nil +} + +// listPages fetches all pages of a paginated OpenStack list endpoint and +// decodes items into T using the given JSON key. +func listPages[T any](ctx context.Context, s *Session, endpoint, key string) ([]T, error) { + var results []T + next := endpoint + for next != "" { + body, err := s.doGet(ctx, next) + if err != nil { + return nil, err + } + var page map[string]json.RawMessage + if err := json.Unmarshal(body, &page); err != nil { + return nil, err + } + var items []T + if err := json.Unmarshal(page[key], &items); err != nil { + return nil, err + } + results = append(results, items...) + next = nextPageURL(body, key) + } + return results, nil +} + +func nextPageURL(body []byte, key string) string { + linksKey := key + "_links" + var page map[string]json.RawMessage + if err := json.Unmarshal(body, &page); err != nil { + return "" + } + raw, ok := page[linksKey] + if !ok { + return "" + } + var links []struct { + Rel string `json:"rel"` + Href string `json:"href"` + } + if err := json.Unmarshal(raw, &links); err != nil { + return "" + } + for _, l := range links { + if l.Rel == "next" { + u, err := url.Parse(l.Href) + if err != nil { + return l.Href + } + // OpenStack sometimes returns http where https is required. + u.Scheme = "https" + return u.String() + } + } + return "" +} diff --git a/internal/openstack/tls.go b/internal/openstack/tls.go new file mode 100644 index 0000000..0ba61ea --- /dev/null +++ b/internal/openstack/tls.go @@ -0,0 +1,19 @@ +package openstack + +import ( + "crypto/tls" + "crypto/x509" + "fmt" +) + +func loadCACert(cfg *tls.Config, cacert string) error { + pool, err := x509.SystemCertPool() + if err != nil { + pool = x509.NewCertPool() + } + if !pool.AppendCertsFromPEM([]byte(cacert)) { + return fmt.Errorf("failed to append CA certificate") + } + cfg.RootCAs = pool + return nil +} diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go new file mode 100644 index 0000000..2a843ad --- /dev/null +++ b/test/e2e/e2e_suite_test.go @@ -0,0 +1,119 @@ +//go:build e2e +// +build e2e + +/* +Copyright 2026. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "fmt" + "os" + "os/exec" + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/azimuth-cloud/cluster-api-janitor-openstack/test/utils" +) + +var ( + // managerImage is the manager image to be built and loaded for testing. + managerImage = "example.com/cluster-api-janitor-openstack:v0.0.1" + // shouldCleanupCertManager tracks whether CertManager was installed by this suite. + shouldCleanupCertManager = false +) + +// TestE2E runs the e2e test suite to validate the solution in an isolated environment. +// The default setup requires Kind and CertManager. +// +// To enable kubectl kuberc (use custom kubectl configurations), set: KUBECTL_KUBERC=true +// By default, kuberc is disabled to ensure consistent test behavior across different environments. +// To skip CertManager installation, set: CERT_MANAGER_INSTALL_SKIP=true +func TestE2E(t *testing.T) { + RegisterFailHandler(Fail) + _, _ = fmt.Fprintf(GinkgoWriter, "Starting cluster-api-janitor-openstack e2e test suite\n") + RunSpecs(t, "e2e suite") +} + +var _ = BeforeSuite(func() { + By("building the manager image") + cmd := exec.Command("make", "docker-build", fmt.Sprintf("IMG=%s", managerImage)) + _, err := utils.Run(cmd) + ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to build the manager image") + + // TODO(user): If you want to change the e2e test vendor from Kind, + // ensure the image is built and available, then remove the following block. + By("loading the manager image on Kind") + err = utils.LoadImageToKindClusterWithName(managerImage) + ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to load the manager image into Kind") + + configureKubectlKubeRC() + setupCertManager() +}) + +var _ = AfterSuite(func() { + teardownCertManager() +}) + +// Disable kubectl kuberc by default for test isolation. +// This prevents local kubectl configurations from affecting test behavior. +// To enable kuberc, set: KUBECTL_KUBERC=true +func configureKubectlKubeRC() { + if os.Getenv("KUBECTL_KUBERC") != "true" { + By("disabling kubectl kuberc for test isolation") + err := os.Setenv("KUBECTL_KUBERC", "false") + ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to disable kubectl kuberc") + _, _ = fmt.Fprintf(GinkgoWriter, + "kubectl kuberc disabled for consistent test behavior (override with KUBECTL_KUBERC=true)\n") + } else { + _, _ = fmt.Fprintf(GinkgoWriter, "kubectl kuberc enabled (KUBECTL_KUBERC=true)\n") + } +} + +// setupCertManager installs CertManager if needed for webhook tests. +// Skips installation if CERT_MANAGER_INSTALL_SKIP=true or if already present. +func setupCertManager() { + if os.Getenv("CERT_MANAGER_INSTALL_SKIP") == "true" { + _, _ = fmt.Fprintf(GinkgoWriter, "Skipping CertManager installation (CERT_MANAGER_INSTALL_SKIP=true)\n") + return + } + + By("checking if CertManager is already installed") + if utils.IsCertManagerCRDsInstalled() { + _, _ = fmt.Fprintf(GinkgoWriter, "CertManager is already installed. Skipping installation.\n") + return + } + + // Mark for cleanup before installation to handle interruptions and partial installs. + shouldCleanupCertManager = true + + By("installing CertManager") + Expect(utils.InstallCertManager()).To(Succeed(), "Failed to install CertManager") +} + +// teardownCertManager uninstalls CertManager if it was installed by setupCertManager. +// This ensures we only remove what we installed. +func teardownCertManager() { + if !shouldCleanupCertManager { + _, _ = fmt.Fprintf(GinkgoWriter, "Skipping CertManager cleanup (not installed by this suite)\n") + return + } + + By("uninstalling CertManager") + utils.UninstallCertManager() +} diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go new file mode 100644 index 0000000..1aa9d47 --- /dev/null +++ b/test/e2e/e2e_test.go @@ -0,0 +1,339 @@ +//go:build e2e +// +build e2e + +/* +Copyright 2026. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "encoding/json" + "fmt" + "os" + "os/exec" + "path/filepath" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/azimuth-cloud/cluster-api-janitor-openstack/test/utils" +) + +// namespace where the project is deployed in +const namespace = "cluster-api-janitor-openstack-system" + +// serviceAccountName created for the project +const serviceAccountName = "cluster-api-janitor-openstack-controller-manager" + +// metricsServiceName is the name of the metrics service of the project +const metricsServiceName = "cluster-api-janitor-openstack-controller-manager-metrics-service" + +// metricsRoleBindingName is the name of the RBAC that will be created to allow get the metrics data +const metricsRoleBindingName = "cluster-api-janitor-openstack-metrics-binding" + +var _ = Describe("Manager", Ordered, func() { + var controllerPodName string + + // Before running the tests, set up the environment by creating the namespace, + // enforce the restricted security policy to the namespace, installing CRDs, + // and deploying the controller. + BeforeAll(func() { + By("creating manager namespace") + cmd := exec.Command("kubectl", "create", "ns", namespace) + _, err := utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "Failed to create namespace") + + By("labeling the namespace to enforce the restricted security policy") + cmd = exec.Command("kubectl", "label", "--overwrite", "ns", namespace, + "pod-security.kubernetes.io/enforce=restricted") + _, err = utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "Failed to label namespace with restricted policy") + + By("installing CRDs") + cmd = exec.Command("make", "install") + _, err = utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "Failed to install CRDs") + + By("deploying the controller-manager") + cmd = exec.Command("make", "deploy", fmt.Sprintf("IMG=%s", managerImage)) + _, err = utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "Failed to deploy the controller-manager") + }) + + // After all tests have been executed, clean up by undeploying the controller, uninstalling CRDs, + // and deleting the namespace. + AfterAll(func() { + By("cleaning up the curl pod for metrics") + cmd := exec.Command("kubectl", "delete", "pod", "curl-metrics", "-n", namespace) + _, _ = utils.Run(cmd) + + By("undeploying the controller-manager") + cmd = exec.Command("make", "undeploy") + _, _ = utils.Run(cmd) + + By("uninstalling CRDs") + cmd = exec.Command("make", "uninstall") + _, _ = utils.Run(cmd) + + By("removing manager namespace") + cmd = exec.Command("kubectl", "delete", "ns", namespace) + _, _ = utils.Run(cmd) + }) + + // After each test, check for failures and collect logs, events, + // and pod descriptions for debugging. + AfterEach(func() { + specReport := CurrentSpecReport() + if specReport.Failed() { + By("Fetching controller manager pod logs") + cmd := exec.Command("kubectl", "logs", controllerPodName, "-n", namespace) + controllerLogs, err := utils.Run(cmd) + if err == nil { + _, _ = fmt.Fprintf(GinkgoWriter, "Controller logs:\n %s", controllerLogs) + } else { + _, _ = fmt.Fprintf(GinkgoWriter, "Failed to get Controller logs: %s", err) + } + + By("Fetching Kubernetes events") + cmd = exec.Command("kubectl", "get", "events", "-n", namespace, "--sort-by=.lastTimestamp") + eventsOutput, err := utils.Run(cmd) + if err == nil { + _, _ = fmt.Fprintf(GinkgoWriter, "Kubernetes events:\n%s", eventsOutput) + } else { + _, _ = fmt.Fprintf(GinkgoWriter, "Failed to get Kubernetes events: %s", err) + } + + By("Fetching curl-metrics logs") + cmd = exec.Command("kubectl", "logs", "curl-metrics", "-n", namespace) + metricsOutput, err := utils.Run(cmd) + if err == nil { + _, _ = fmt.Fprintf(GinkgoWriter, "Metrics logs:\n %s", metricsOutput) + } else { + _, _ = fmt.Fprintf(GinkgoWriter, "Failed to get curl-metrics logs: %s", err) + } + + By("Fetching controller manager pod description") + cmd = exec.Command("kubectl", "describe", "pod", controllerPodName, "-n", namespace) + podDescription, err := utils.Run(cmd) + if err == nil { + fmt.Println("Pod description:\n", podDescription) + } else { + fmt.Println("Failed to describe controller pod") + } + } + }) + + SetDefaultEventuallyTimeout(2 * time.Minute) + SetDefaultEventuallyPollingInterval(time.Second) + + Context("Manager", func() { + It("should run successfully", func() { + By("validating that the controller-manager pod is running as expected") + verifyControllerUp := func(g Gomega) { + By("getting the name of the controller-manager pod") + cmd := exec.Command("kubectl", "get", + "pods", "-l", "control-plane=controller-manager", + "-o", "go-template={{ range .items }}"+ + "{{ if not .metadata.deletionTimestamp }}"+ + "{{ .metadata.name }}"+ + "{{ \"\\n\" }}{{ end }}{{ end }}", + "-n", namespace, + ) + + podOutput, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred(), "Failed to retrieve controller-manager pod information") + podNames := utils.GetNonEmptyLines(podOutput) + g.Expect(podNames).To(HaveLen(1), "expected 1 controller pod running") + controllerPodName = podNames[0] + g.Expect(controllerPodName).To(ContainSubstring("controller-manager")) + + By("validating the pod's status") + cmd = exec.Command("kubectl", "get", + "pods", controllerPodName, "-o", "jsonpath={.status.phase}", + "-n", namespace, + ) + output, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(output).To(Equal("Running"), "Incorrect controller-manager pod status") + } + Eventually(verifyControllerUp).Should(Succeed()) + }) + + It("should ensure the metrics endpoint is serving metrics", func() { + By("creating a ClusterRoleBinding for the service account to allow access to metrics") + cmd := exec.Command("kubectl", "create", "clusterrolebinding", metricsRoleBindingName, + "--clusterrole=cluster-api-janitor-openstack-metrics-reader", + fmt.Sprintf("--serviceaccount=%s:%s", namespace, serviceAccountName), + ) + _, err := utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "Failed to create ClusterRoleBinding") + + By("validating that the metrics service is available") + cmd = exec.Command("kubectl", "get", "service", metricsServiceName, "-n", namespace) + _, err = utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "Metrics service should exist") + + By("getting the service account token") + token, err := serviceAccountToken() + Expect(err).NotTo(HaveOccurred()) + Expect(token).NotTo(BeEmpty()) + + By("ensuring the controller pod is ready") + verifyControllerPodReady := func(g Gomega) { + cmd := exec.Command("kubectl", "get", "pod", controllerPodName, "-n", namespace, + "-o", "jsonpath={.status.conditions[?(@.type=='Ready')].status}") + output, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(output).To(Equal("True"), "Controller pod not ready") + } + Eventually(verifyControllerPodReady, 3*time.Minute, time.Second).Should(Succeed()) + + By("verifying that the controller manager is serving the metrics server") + verifyMetricsServerStarted := func(g Gomega) { + cmd := exec.Command("kubectl", "logs", controllerPodName, "-n", namespace) + output, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(output).To(ContainSubstring("Serving metrics server"), + "Metrics server not yet started") + } + Eventually(verifyMetricsServerStarted, 3*time.Minute, time.Second).Should(Succeed()) + + // +kubebuilder:scaffold:e2e-metrics-webhooks-readiness + + By("creating the curl-metrics pod to access the metrics endpoint") + cmd = exec.Command("kubectl", "run", "curl-metrics", "--restart=Never", + "--namespace", namespace, + "--image=curlimages/curl:latest", + "--overrides", + fmt.Sprintf(`{ + "spec": { + "containers": [{ + "name": "curl", + "image": "curlimages/curl:latest", + "command": ["/bin/sh", "-c"], + "args": [ + "for i in $(seq 1 30); do curl -v -k -H 'Authorization: Bearer %s' https://%s.%s.svc.cluster.local:8443/metrics && exit 0 || sleep 2; done; exit 1" + ], + "securityContext": { + "readOnlyRootFilesystem": true, + "allowPrivilegeEscalation": false, + "capabilities": { + "drop": ["ALL"] + }, + "runAsNonRoot": true, + "runAsUser": 1000, + "seccompProfile": { + "type": "RuntimeDefault" + } + } + }], + "serviceAccountName": "%s" + } + }`, token, metricsServiceName, namespace, serviceAccountName)) + _, err = utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "Failed to create curl-metrics pod") + + By("waiting for the curl-metrics pod to complete.") + verifyCurlUp := func(g Gomega) { + cmd := exec.Command("kubectl", "get", "pods", "curl-metrics", + "-o", "jsonpath={.status.phase}", + "-n", namespace) + output, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(output).To(Equal("Succeeded"), "curl pod in wrong status") + } + Eventually(verifyCurlUp, 5*time.Minute).Should(Succeed()) + + By("getting the metrics by checking curl-metrics logs") + verifyMetricsAvailable := func(g Gomega) { + metricsOutput, err := getMetricsOutput() + g.Expect(err).NotTo(HaveOccurred(), "Failed to retrieve logs from curl pod") + g.Expect(metricsOutput).NotTo(BeEmpty()) + g.Expect(metricsOutput).To(ContainSubstring("< HTTP/1.1 200 OK")) + } + Eventually(verifyMetricsAvailable, 2*time.Minute).Should(Succeed()) + }) + + // +kubebuilder:scaffold:e2e-webhooks-checks + + // TODO: Customize the e2e test suite with scenarios specific to your project. + // Consider applying sample/CR(s) and check their status and/or verifying + // the reconciliation by using the metrics, i.e.: + // metricsOutput, err := getMetricsOutput() + // Expect(err).NotTo(HaveOccurred(), "Failed to retrieve logs from curl pod") + // Expect(metricsOutput).To(ContainSubstring( + // fmt.Sprintf(`controller_runtime_reconcile_total{controller="%s",result="success"} 1`, + // strings.ToLower(), + // )) + }) +}) + +// serviceAccountToken returns a token for the specified service account in the given namespace. +// It uses the Kubernetes TokenRequest API to generate a token by directly sending a request +// and parsing the resulting token from the API response. +func serviceAccountToken() (string, error) { + const tokenRequestRawString = `{ + "apiVersion": "authentication.k8s.io/v1", + "kind": "TokenRequest" + }` + + By("creating temporary file to store the token request") + secretName := fmt.Sprintf("%s-token-request", serviceAccountName) + tokenRequestFile := filepath.Join("/tmp", secretName) + err := os.WriteFile(tokenRequestFile, []byte(tokenRequestRawString), os.FileMode(0o644)) + if err != nil { + return "", err + } + + var out string + verifyTokenCreation := func(g Gomega) { + By("executing kubectl command to create the token") + cmd := exec.Command("kubectl", "create", "--raw", fmt.Sprintf( + "/api/v1/namespaces/%s/serviceaccounts/%s/token", + namespace, + serviceAccountName, + ), "-f", tokenRequestFile) + + output, err := cmd.CombinedOutput() + g.Expect(err).NotTo(HaveOccurred()) + + By("parsing the JSON output to extract the token") + var token tokenRequest + err = json.Unmarshal(output, &token) + g.Expect(err).NotTo(HaveOccurred()) + + out = token.Status.Token + } + Eventually(verifyTokenCreation).Should(Succeed()) + + return out, err +} + +// getMetricsOutput retrieves and returns the logs from the curl pod used to access the metrics endpoint. +func getMetricsOutput() (string, error) { + By("getting the curl-metrics logs") + cmd := exec.Command("kubectl", "logs", "curl-metrics", "-n", namespace) + return utils.Run(cmd) +} + +// tokenRequest is a simplified representation of the Kubernetes TokenRequest API response, +// containing only the token field that we need to extract. +type tokenRequest struct { + Status struct { + Token string `json:"token"` + } `json:"status"` +} diff --git a/test/utils/utils.go b/test/utils/utils.go new file mode 100644 index 0000000..a408630 --- /dev/null +++ b/test/utils/utils.go @@ -0,0 +1,226 @@ +/* +Copyright 2026. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package utils + +import ( + "bufio" + "bytes" + "fmt" + "os" + "os/exec" + "strings" + + . "github.com/onsi/ginkgo/v2" // nolint:revive,staticcheck +) + +const ( + certmanagerVersion = "v1.20.2" + certmanagerURLTmpl = "https://github.com/cert-manager/cert-manager/releases/download/%s/cert-manager.yaml" + + defaultKindBinary = "kind" + defaultKindCluster = "kind" +) + +func warnError(err error) { + _, _ = fmt.Fprintf(GinkgoWriter, "warning: %v\n", err) +} + +// Run executes the provided command within this context +func Run(cmd *exec.Cmd) (string, error) { + dir, _ := GetProjectDir() + cmd.Dir = dir + + if err := os.Chdir(cmd.Dir); err != nil { + _, _ = fmt.Fprintf(GinkgoWriter, "chdir dir: %q\n", err) + } + + cmd.Env = append(os.Environ(), "GO111MODULE=on") + command := strings.Join(cmd.Args, " ") + _, _ = fmt.Fprintf(GinkgoWriter, "running: %q\n", command) + output, err := cmd.CombinedOutput() + if err != nil { + return string(output), fmt.Errorf("%q failed with error %q: %w", command, string(output), err) + } + + return string(output), nil +} + +// UninstallCertManager uninstalls the cert manager +func UninstallCertManager() { + url := fmt.Sprintf(certmanagerURLTmpl, certmanagerVersion) + cmd := exec.Command("kubectl", "delete", "-f", url) + if _, err := Run(cmd); err != nil { + warnError(err) + } + + // Delete leftover leases in kube-system (not cleaned by default) + kubeSystemLeases := []string{ + "cert-manager-cainjector-leader-election", + "cert-manager-controller", + } + for _, lease := range kubeSystemLeases { + cmd = exec.Command("kubectl", "delete", "lease", lease, + "-n", "kube-system", "--ignore-not-found", "--force", "--grace-period=0") + if _, err := Run(cmd); err != nil { + warnError(err) + } + } +} + +// InstallCertManager installs the cert manager bundle. +func InstallCertManager() error { + url := fmt.Sprintf(certmanagerURLTmpl, certmanagerVersion) + cmd := exec.Command("kubectl", "apply", "-f", url) + if _, err := Run(cmd); err != nil { + return err + } + // Wait for cert-manager-webhook to be ready, which can take time if cert-manager + // was re-installed after uninstalling on a cluster. + cmd = exec.Command("kubectl", "wait", "deployment.apps/cert-manager-webhook", + "--for", "condition=Available", + "--namespace", "cert-manager", + "--timeout", "5m", + ) + + _, err := Run(cmd) + return err +} + +// IsCertManagerCRDsInstalled checks if any Cert Manager CRDs are installed +// by verifying the existence of key CRDs related to Cert Manager. +func IsCertManagerCRDsInstalled() bool { + // List of common Cert Manager CRDs + certManagerCRDs := []string{ + "certificates.cert-manager.io", + "issuers.cert-manager.io", + "clusterissuers.cert-manager.io", + "certificaterequests.cert-manager.io", + "orders.acme.cert-manager.io", + "challenges.acme.cert-manager.io", + } + + // Execute the kubectl command to get all CRDs + cmd := exec.Command("kubectl", "get", "crds") + output, err := Run(cmd) + if err != nil { + return false + } + + // Check if any of the Cert Manager CRDs are present + crdList := GetNonEmptyLines(output) + for _, crd := range certManagerCRDs { + for _, line := range crdList { + if strings.Contains(line, crd) { + return true + } + } + } + + return false +} + +// LoadImageToKindClusterWithName loads a local docker image to the kind cluster +func LoadImageToKindClusterWithName(name string) error { + cluster := defaultKindCluster + if v, ok := os.LookupEnv("KIND_CLUSTER"); ok { + cluster = v + } + kindOptions := []string{"load", "docker-image", name, "--name", cluster} + kindBinary := defaultKindBinary + if v, ok := os.LookupEnv("KIND"); ok { + kindBinary = v + } + cmd := exec.Command(kindBinary, kindOptions...) + _, err := Run(cmd) + return err +} + +// GetNonEmptyLines converts given command output string into individual objects +// according to line breakers, and ignores the empty elements in it. +func GetNonEmptyLines(output string) []string { + var res []string + elements := strings.SplitSeq(output, "\n") + for element := range elements { + if element != "" { + res = append(res, element) + } + } + + return res +} + +// GetProjectDir will return the directory where the project is +func GetProjectDir() (string, error) { + wd, err := os.Getwd() + if err != nil { + return wd, fmt.Errorf("failed to get current working directory: %w", err) + } + wd = strings.ReplaceAll(wd, "/test/e2e", "") + return wd, nil +} + +// UncommentCode searches for target in the file and remove the comment prefix +// of the target content. The target content may span multiple lines. +func UncommentCode(filename, target, prefix string) error { + // false positive + // nolint:gosec + content, err := os.ReadFile(filename) + if err != nil { + return fmt.Errorf("failed to read file %q: %w", filename, err) + } + strContent := string(content) + + idx := strings.Index(strContent, target) + if idx < 0 { + return fmt.Errorf("unable to find the code %q to be uncommented", target) + } + + out := new(bytes.Buffer) + _, err = out.Write(content[:idx]) + if err != nil { + return fmt.Errorf("failed to write to output: %w", err) + } + + scanner := bufio.NewScanner(bytes.NewBufferString(target)) + if !scanner.Scan() { + return nil + } + for { + if _, err = out.WriteString(strings.TrimPrefix(scanner.Text(), prefix)); err != nil { + return fmt.Errorf("failed to write to output: %w", err) + } + // Avoid writing a newline in case the previous line was the last in target. + if !scanner.Scan() { + break + } + if _, err = out.WriteString("\n"); err != nil { + return fmt.Errorf("failed to write to output: %w", err) + } + } + + if _, err = out.Write(content[idx+len(target):]); err != nil { + return fmt.Errorf("failed to write to output: %w", err) + } + + // false positive + // nolint:gosec + if err = os.WriteFile(filename, out.Bytes(), 0644); err != nil { + return fmt.Errorf("failed to write file %q: %w", filename, err) + } + + return nil +} From 7489512f2b0ac80ade096c8c9919c0fe4cc6c06c Mon Sep 17 00:00:00 2001 From: Mathieu Grzybek Date: Tue, 7 Jul 2026 17:07:26 +0200 Subject: [PATCH 02/27] =?UTF-8?q?feat(openstack):=20Epic=201=20=E2=80=94?= =?UTF-8?q?=20authentification=20OpenStack=20(TDD)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Tests for US1.1 (v3applicationcredential), US1.2 (interface/region), US1.3 (revoked credential / catalog 404), US1.4 (custom CA) - Bug : s.userID not assigned in getToken - Remplacing custom errors_as by errors.As stdlib - Adding Session.HasEndpoint() for test introspection - 24 tests, covering cloud.go 85–100% Signed-off-by: Mathieu Grzybek --- AGENTS.md | 320 +++++++++++++++ ROADMAP.md | 649 +++++++++++++++++++++++++++++++ internal/openstack/cloud.go | 25 +- internal/openstack/cloud_test.go | 631 ++++++++++++++++++++++++++++++ 4 files changed, 1614 insertions(+), 11 deletions(-) create mode 100644 AGENTS.md create mode 100644 ROADMAP.md create mode 100644 internal/openstack/cloud_test.go diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..ab50558 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,320 @@ +# cluster-api-janitor-openstack - AI Agent Guide + +## Project Structure + +**Single-group layout (default):** +``` +cmd/main.go Manager entry (registers controllers/webhooks) +api//*_types.go CRD schemas (+kubebuilder markers) +api//zz_generated.* Auto-generated (DO NOT EDIT) +internal/controller/* Reconciliation logic +internal/webhook/* Validation/defaulting (if present) +config/crd/bases/* Generated CRDs (DO NOT EDIT) +config/rbac/role.yaml Generated RBAC (DO NOT EDIT) +config/samples/* Example CRs (edit these) +Makefile Build/test/deploy commands +PROJECT Kubebuilder metadata Auto-generated (DO NOT EDIT) +``` + +**Multi-group layout** (for projects with multiple API groups): +``` +api///*_types.go CRD schemas by group +internal/controller//* Controllers by group +internal/webhook///* Webhooks by group and version (if present) +``` + +Multi-group layout organizes APIs by group name (e.g., `batch`, `apps`). Check the `PROJECT` file for `multigroup: true`. + +**To convert to multi-group layout:** +1. Run: `kubebuilder edit --multigroup=true` +2. Move APIs: `mkdir -p api/ && mv api/ api//` +3. Move controllers: `mkdir -p internal/controller/ && mv internal/controller/*.go internal/controller//` +4. Move webhooks (if present): `mkdir -p internal/webhook/ && mv internal/webhook/ internal/webhook//` +5. Update import paths in all files +6. Fix `path` in `PROJECT` file for each resource +7. Update test suite CRD paths (add one more `..` to relative paths) + +## Critical Rules + +### Never Edit These (Auto-Generated) +- `config/crd/bases/*.yaml` - from `make manifests` +- `config/rbac/role.yaml` - from `make manifests` +- `config/webhook/manifests.yaml` - from `make manifests` +- `**/zz_generated.*.go` - from `make generate` +- `PROJECT` - from `kubebuilder [OPTIONS]` + +### Never Remove Scaffold Markers +Do NOT delete `// +kubebuilder:scaffold:*` comments. CLI injects code at these markers. + +### Keep Project Structure +Do not move files around. The CLI expects files in specific locations. + +### Always Use CLI Commands +Always use `kubebuilder create api` and `kubebuilder create webhook` to scaffold. Do NOT create files manually. + +### E2E Tests Require an Isolated Kind Cluster +The e2e tests are designed to validate the solution in an isolated environment (similar to GitHub Actions CI). +Ensure you run them against a dedicated [Kind](https://kind.sigs.k8s.io/) cluster (not your “real” dev/prod cluster). + +## After Making Changes + +**After editing `*_types.go` or markers:** +``` +make manifests # Regenerate CRDs/RBAC from markers +make generate # Regenerate DeepCopy methods +``` + +**After editing `*.go` files:** +``` +make lint-fix # Auto-fix code style +make test # Run unit tests +``` + +## CLI Commands Cheat Sheet + +### Create API (your own types) +```bash +kubebuilder create api --group --version --kind +``` + +### Deploy Image Plugin (scaffold to deploy/manage ANY container image) + +Generate a controller that deploys and manages a container image (nginx, redis, memcached, your app, etc.): + +```bash +# Example: deploying memcached +kubebuilder create api --group example.com --version v1alpha1 --kind Memcached \ + --image=memcached:alpine \ + --plugins=deploy-image.go.kubebuilder.io/v1-alpha +``` + +Scaffolds good-practice code: reconciliation logic, status conditions, finalizers, RBAC. Use as a reference implementation. + + +### Create Webhooks +```bash +# Validation + defaulting +kubebuilder create webhook --group --version --kind \ + --defaulting --programmatic-validation + +# Conversion webhook (for multi-version APIs) +kubebuilder create webhook --group --version v1 --kind \ + --conversion --spoke v2 +``` + +### Controller for Core Kubernetes Types +```bash +# Watch Pods +kubebuilder create api --group core --version v1 --kind Pod \ + --controller=true --resource=false + +# Watch Deployments +kubebuilder create api --group apps --version v1 --kind Deployment \ + --controller=true --resource=false +``` + +### Controller for External Types (e.g., from other operators) + +Watch resources from external APIs (cert-manager, Argo CD, Istio, etc.): + +```bash +# Example: watching cert-manager Certificate resources +kubebuilder create api \ + --group cert-manager --version v1 --kind Certificate \ + --controller=true --resource=false \ + --external-api-path=github.com/cert-manager/cert-manager/pkg/apis/certmanager/v1 \ + --external-api-domain=io \ + --external-api-module=github.com/cert-manager/cert-manager +``` + +**Note:** Use `--external-api-module=@` only if you need a specific version. Otherwise, omit `@` to use what's in go.mod. + +### Webhook for External Types + +```bash +# Example: validating external resources +kubebuilder create webhook \ + --group cert-manager --version v1 --kind Issuer \ + --defaulting \ + --external-api-path=github.com/cert-manager/cert-manager/pkg/apis/certmanager/v1 \ + --external-api-domain=io \ + --external-api-module=github.com/cert-manager/cert-manager +``` + +## Testing & Development + +```bash +make test # Run unit tests (uses envtest: real K8s API + etcd) +make run # Run locally (uses current kubeconfig context) +``` + +Tests use **Ginkgo + Gomega** (BDD style). Check `suite_test.go` for setup. + +## Deployment Workflow + +```bash +# 1. Regenerate manifests +make manifests generate + +# 2. Build & deploy +export IMG=/:tag +make docker-build docker-push IMG=$IMG # Or: kind load docker-image $IMG --name +make deploy IMG=$IMG + +# 3. Test +kubectl apply -k config/samples/ + +# 4. Debug +kubectl logs -n -system deployment/-controller-manager -c manager -f +``` + +### API Design + +**Key markers for** `api//*_types.go`: + +```go +// +kubebuilder:object:root=true +// +kubebuilder:subresource:status +// +kubebuilder:resource:scope=Namespaced +// +kubebuilder:printcolumn:name="Status",type=string,JSONPath=".status.conditions[?(@.type=='Ready')].status" + +// On fields: +// +kubebuilder:validation:Required +// +kubebuilder:validation:Minimum=1 +// +kubebuilder:validation:MaxLength=100 +// +kubebuilder:validation:Pattern="^[a-z]+$" +// +kubebuilder:default="value" +``` + +- **Use** `metav1.Condition` for status (not custom string fields) +- **Use predefined types**: `metav1.Time` instead of `string` for dates +- **Follow K8s API conventions**: Standard field names (`spec`, `status`, `metadata`) + +### Controller Design + +**RBAC markers in** `internal/controller/*_controller.go`: + +```go +// +kubebuilder:rbac:groups=mygroup.example.com,resources=mykinds,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=mygroup.example.com,resources=mykinds/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=mygroup.example.com,resources=mykinds/finalizers,verbs=update +// +kubebuilder:rbac:groups=events.k8s.io,resources=events,verbs=create;patch +// +kubebuilder:rbac:groups=apps,resources=deployments,verbs=get;list;watch;create;update;patch;delete +``` + +**Implementation rules:** +- **Idempotent reconciliation**: Safe to run multiple times +- **Re-fetch before updates**: `r.Get(ctx, req.NamespacedName, obj)` before `r.Update` to avoid conflicts +- **Structured logging**: `log := log.FromContext(ctx); log.Info("msg", "key", val)` +- **Owner references**: Enable automatic garbage collection (`SetControllerReference`) +- **Watch secondary resources**: Use `.Owns()` or `.Watches()`, not just `RequeueAfter` +- **Finalizers**: Clean up external resources (buckets, VMs, DNS entries) + +### Logging + +**Follow Kubernetes logging message style guidelines:** + +- Start from a capital letter +- Do not end the message with a period +- Active voice: subject present (`"Deployment could not create Pod"`) or omitted (`"Could not create Pod"`) +- Past tense: `"Could not delete Pod"` not `"Cannot delete Pod"` +- Specify object type: `"Deleted Pod"` not `"Deleted"` +- Balanced key-value pairs + +```go +log.Info("Starting reconciliation") +log.Info("Created Deployment", "name", deploy.Name) +log.Error(err, "Failed to create Pod", "name", name) +``` + +**Reference:** https://github.com/kubernetes/community/blob/master/contributors/devel/sig-instrumentation/logging.md#message-style-guidelines + +### Webhooks +- **Create all types together**: `--defaulting --programmatic-validation --conversion` +- **When`--force`is used**: Backup custom logic first, then restore after scaffolding +- **For multi-version APIs**: Use hub-and-spoke pattern (`--conversion --spoke v2`) + - Hub version: Usually oldest stable version (v1) + - Spoke versions: Newer versions that convert to/from hub (v2, v3) + - Example: `--group crew --version v1 --kind Captain --conversion --spoke v2` (v1 is hub, v2 is spoke) + +### Learning from Examples + +The **deploy-image plugin** scaffolds a complete controller following good practices. Use it as a reference implementation: + +```bash +kubebuilder create api --group example --version v1alpha1 --kind MyApp \ + --image= --plugins=deploy-image.go.kubebuilder.io/v1-alpha +``` + +Generated code includes: status conditions (`metav1.Condition`), finalizers, owner references, events, idempotent reconciliation. + +## Distribution Options + +### Option 1: YAML Bundle (Kustomize) + +```bash +# Generate dist/install.yaml from Kustomize manifests +make build-installer IMG=/:tag +``` + +**Key points:** +- The `dist/install.yaml` is generated from Kustomize manifests (CRDs, RBAC, Deployment) +- Commit this file to your repository for easy distribution +- Users only need `kubectl` to install (no additional tools required) + +**Example:** Users install with a single command: +```bash +kubectl apply -f https://raw.githubusercontent.com////dist/install.yaml +``` + +### Option 2: Helm Chart + +```bash +kubebuilder edit --plugins=helm/v2-alpha # Generates dist/chart/ (default) +kubebuilder edit --plugins=helm/v2-alpha --output-dir=charts # Generates charts/chart/ +``` + +**For development:** +```bash +make helm-deploy IMG=/: # Deploy manager via Helm +make helm-deploy IMG=$IMG HELM_EXTRA_ARGS="--set ..." # Deploy with custom values +make helm-status # Show release status +make helm-uninstall # Remove release +make helm-history # View release history +make helm-rollback # Rollback to previous version +``` + +**For end users/production:** +```bash +helm install my-release .//chart/ --namespace --create-namespace +``` + +**Important:** If you add webhooks or modify manifests after initial chart generation: +1. Backup any customizations in `/chart/values.yaml` and `/chart/manager/manager.yaml` +2. Re-run: `kubebuilder edit --plugins=helm/v2-alpha --force` (use same `--output-dir` if customized) +3. Manually restore your custom values from the backup + +### Publish Container Image + +```bash +export IMG=/: +make docker-build docker-push IMG=$IMG +``` + +## References + +### Essential Reading +- **Kubebuilder Book**: https://book.kubebuilder.io (comprehensive guide) +- **controller-runtime FAQ**: https://github.com/kubernetes-sigs/controller-runtime/blob/main/FAQ.md (common patterns and questions) +- **Good Practices**: https://book.kubebuilder.io/reference/good-practices.html (why reconciliation is idempotent, status conditions, etc.) +- **Logging Conventions**: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-instrumentation/logging.md#message-style-guidelines (message style, verbosity levels) + +### API Design & Implementation +- **API Conventions**: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md +- **Operator Pattern**: https://kubernetes.io/docs/concepts/extend-kubernetes/operator/ +- **Markers Reference**: https://book.kubebuilder.io/reference/markers.html + +### Tools & Libraries +- **controller-runtime**: https://github.com/kubernetes-sigs/controller-runtime +- **controller-tools**: https://github.com/kubernetes-sigs/controller-tools +- **Kubebuilder Repo**: https://github.com/kubernetes-sigs/kubebuilder diff --git a/ROADMAP.md b/ROADMAP.md new file mode 100644 index 0000000..3f9a654 --- /dev/null +++ b/ROADMAP.md @@ -0,0 +1,649 @@ +# Réécriture du projet en Go + +## Contexte + +Le projet actuel est écrit en Python (asyncio + kopf + easykube + httpx). C'est un opérateur Kubernetes qui nettoie les ressources OpenStack laissées par l'OCCM et le CSI Cinder lors de la suppression de clusters Cluster API. + +La réécriture suit la méthode TDD : les tests Gherkin sont écrits en premier, puis les implémentations. + +Outil de scaffolding : **kubebuilder** (skill disponible). + +--- + +## Audit du code Python existant + +### Modules principaux + +| Fichier | Rôle | +|---|---| +| `capi_janitor/openstack/openstack.py` | Client OpenStack : authentification, catalogue de services, ressources REST paginées | +| `capi_janitor/openstack/operator.py` | Logique opérateur : handlers kopf, filtres de ressources, purge OpenStack | + +### Fonctionnalités couvertes + +**Authentification OpenStack** +- Uniquement `v3applicationcredential` +- Gestion du token X-Auth-Token (refresh avec mutex asyncio) +- Support certificat CA personnalisé (cacert depuis le secret K8s) +- Catalogue de services filtré par interface (public/internal/admin) et région + +**Filtrage des ressources à nettoyer** +- Floating IPs : description `"Floating IP for Kubernetes external service … from cluster "` +- Load Balancers Octavia : nom `kube_service__*` +- Security Groups : description `"Security Group for Service LoadBalancer in cluster "` +- Volumes Cinder : métadonnée `cinder.csi.openstack.org/cluster == `, sauf propriété `janitor.capi.azimuth-cloud.com/keep == true` +- Snapshots Cinder : même métadonnée cluster + +**Politique de suppression** +- Volumes : configurable via env var `CAPI_JANITOR_DEFAULT_VOLUMES_POLICY` (défaut `delete`) et annotation `janitor.capi.stackhpc.com/volumes-policy` par cluster +- Application Credential : supprimé si annotation `janitor.capi.stackhpc.com/credential-policy: delete` sur le secret ET si c'est le dernier finalizer + +**Lifecycle Kubernetes** +- Finalizer `janitor.capi.stackhpc.com` sur `OpenStackCluster` +- Nom du cluster : label `cluster.x-k8s.io/cluster-name` en priorité, sinon `metadata.name` +- Retry via annotation aléatoire `janitor.capi.stackhpc.com/retry` (déclenche un nouvel événement) +- Backoff configurable `CAPI_JANITOR_RETRY_DEFAULT_DELAY` (défaut 60s) + +**Gestion des erreurs** +- HTTP 400/409 lors de la suppression : retry silencieux +- HTTP 404 lors de la récupération du catalogue : authentification considérée comme échouée (sans erreur fatale) +- HTTP 422 lors du patch des finalizers : `TemporaryError` kopf +- Erreur catalogue `volumev3` → fallback sur `block-storage` + +### Tests existants + +| Fichier | Ce qui est testé | +|---|---| +| `test_openstack.py` | Authentification réussie, 404, absence d'interface, absence de région, multiples services | +| `test_operator.py` | Filtrage FIPs, LBs, SGs, volumes, snapshots ; `empty()` ; `try_delete()` ; handler d'événement (ajout finalizer, skip, purge) ; erreur d'auth dans purge | + +**Lacune notable** : `test_purge_openstack_resources_success` est commenté (complexité du mock). + +### Chart Helm + +- `ClusterRole` : namespaces (list/watch), events (create), secrets (get/delete), openstackclusters (list/get/watch/patch), CRDs (list/get/watch) +- Valeur `defaultVolumesPolicy: delete` +- Image : `ghcr.io/azimuth-cloud/cluster-api-janitor-openstack` + +### PRs en attente à intégrer + +| PR | Titre | Impact | +|---|---|---| +| #261 | Fix leaving Azimuth cluster loadbalancers behind | Ajoute la détection des LBs Azimuth (`kube_service__` + LBs nommés différemment par Azimuth) | + +--- + +## Feuille de route agile + +--- + +### Epic 1 — Authentification OpenStack + +#### US1.1 — Authentification via Application Credential v3 + +```gherkin +Feature: Authentification OpenStack via Application Credential + In order to accéder aux APIs OpenStack + As an opérateur + I want to m'authentifier avec un Application Credential v3 + + Scenario: Authentification réussie + Given un clouds.yaml avec auth_type "v3applicationcredential" + And un application_credential_id et application_credential_secret valides + When l'opérateur initialise la connexion OpenStack + Then un token X-Auth-Token est obtenu depuis Keystone + And le catalogue de services est chargé + + Scenario: Refresh du token lors d'une expiration + Given un token X-Auth-Token expiré + When l'opérateur effectue un appel API + Then un nouveau token est demandé à Keystone + And l'appel original est rejoué avec le nouveau token + + Scenario: Authentification avec un type non supporté + Given un clouds.yaml avec auth_type "password" + When l'opérateur tente de créer un client Cloud + Then une erreur UnsupportedAuthenticationError est levée +``` + +#### US1.2 — Filtrage du catalogue de services par interface et région + +```gherkin +Feature: Catalogue de services OpenStack + Scenario: Endpoint sélectionné selon l'interface configurée + Given un catalogue avec des endpoints "public" et "internal" + And l'interface configurée est "public" + When le catalogue est chargé + Then seuls les endpoints "public" sont retenus + + Scenario: Endpoint sélectionné selon la région configurée + Given un catalogue avec des endpoints pour "RegionOne" et "RegionTwo" + And la région configurée est "RegionOne" + When le catalogue est chargé + Then seuls les endpoints de "RegionOne" sont retenus + + Scenario: Aucune région configurée + Given un catalogue avec des endpoints dans plusieurs régions + And aucune région n'est configurée + When le catalogue est chargé + Then le premier endpoint correspondant à l'interface est retenu pour chaque service +``` + +#### US1.3 — Gestion d'un credential révoqué ou invalide + +```gherkin +Feature: Credential OpenStack invalide + Scenario: Application credential supprimé avant la purge + Given un cluster OpenStack en cours de suppression + And l'application credential a déjà été supprimé + When l'opérateur tente de s'authentifier + Then is_authenticated retourne false + And si include_appcred est true, un warning est émis et la purge s'arrête proprement + And si include_appcred est false, une AuthenticationError est levée + + Scenario: Catalogue retourne 404 + Given une URL Keystone valide mais le catalogue retourne 404 + When l'opérateur charge le catalogue + Then is_authenticated retourne false + And aucune erreur fatale n'est levée +``` + +#### US1.4 — Support des certificats CA personnalisés + +```gherkin +Feature: Certificat CA personnalisé + Scenario: CA fourni dans le secret Kubernetes + Given un secret Kubernetes contenant une entrée "cacert" + When l'opérateur initialise le transport TLS + Then le CA est chargé dans le contexte SSL + And les appels HTTPS vers OpenStack utilisent ce CA pour la vérification + + Scenario: Pas de CA fourni + Given un secret Kubernetes sans entrée "cacert" + When l'opérateur initialise le transport TLS + Then le CA système est utilisé pour la vérification TLS +``` + +--- + +### Epic 2 — Nettoyage des Floating IPs + +#### US2.1 — Identifier les Floating IPs d'un cluster + +```gherkin +Feature: Identification des Floating IPs d'un cluster + Scenario: FIP appartenant au cluster + Given une liste de Floating IPs OpenStack + And une FIP avec la description "Floating IP for Kubernetes external service from cluster mycluster" + When les FIPs du cluster "mycluster" sont listées + Then cette FIP est incluse dans le résultat + + Scenario: FIP d'un autre cluster + Given une FIP avec la description "Floating IP for Kubernetes external service from cluster othercluster" + When les FIPs du cluster "mycluster" sont listées + Then cette FIP est exclue du résultat + + Scenario: FIP sans description Kubernetes + Given une FIP avec la description "Some other description" + When les FIPs du cluster "mycluster" sont listées + Then cette FIP est exclue du résultat +``` + +#### US2.2 — Supprimer les Floating IPs + +```gherkin +Feature: Suppression des Floating IPs + Scenario: Suppression réussie + Given une FIP appartenant au cluster "mycluster" + When la purge des FIPs est déclenchée + Then la FIP est supprimée via l'API Neutron + And un log INFO est émis + + Scenario: Erreur HTTP 400 lors de la suppression + Given une suppression de FIP retourne HTTP 400 + When la purge tente de supprimer la FIP + Then un warning est émis + And la suppression continue pour les autres FIPs + And check_fips est true pour déclencher une vérification + + Scenario: Erreur HTTP 500 lors de la suppression + Given une suppression de FIP retourne HTTP 500 + When la purge tente de supprimer la FIP + Then une exception est propagée +``` + +--- + +### Epic 3 — Nettoyage des Load Balancers Octavia + +#### US3.1 — Identifier les Load Balancers Kubernetes d'un cluster + +```gherkin +Feature: Identification des Load Balancers Kubernetes + Scenario: LB appartenant au cluster + Given un LB avec le nom "kube_service_mycluster_api" + When les LBs du cluster "mycluster" sont listés + Then ce LB est inclus dans le résultat + + Scenario: LB d'un autre cluster + Given un LB avec le nom "kube_service_othercluster_api" + When les LBs du cluster "mycluster" sont listés + Then ce LB est exclu du résultat + + Scenario: LB sans préfixe kube_service + Given un LB avec le nom "fake_service_mycluster_api" + When les LBs du cluster "mycluster" sont listés + Then ce LB est exclu du résultat +``` + +#### US3.2 — Identifier les Load Balancers Azimuth (PR #261) + +```gherkin +Feature: Identification des Load Balancers Azimuth + Scenario: LB Azimuth appartenant au cluster + Given un LB Azimuth identifiable comme appartenant au cluster "mycluster" + When les LBs du cluster "mycluster" sont listés + Then ce LB Azimuth est inclus dans le résultat + + Scenario: Erreur HTTP lors du listing des LBs + Given l'API Octavia retourne une erreur HTTP lors du listing + When les LBs du cluster "mycluster" sont listés + Then un log ERROR est émis avec le code HTTP + And aucune exception n'est propagée + And un warning indique que des LBs pourraient rester +``` + +#### US3.3 — Supprimer les Load Balancers en cascade + +```gherkin +Feature: Suppression des Load Balancers en cascade + Scenario: Suppression réussie avec cascade + Given un LB appartenant au cluster "mycluster" + When la purge des LBs est déclenchée + Then le LB est supprimé avec le paramètre cascade=true + And les ressources Octavia associées (listeners, pools, membres) sont supprimées +``` + +--- + +### Epic 4 — Nettoyage des Security Groups + +#### US4.1 — Identifier les Security Groups d'un cluster + +```gherkin +Feature: Identification des Security Groups d'un cluster + Scenario: SG appartenant au cluster + Given un SG avec la description "Security Group for Service LoadBalancer in cluster mycluster" + When les SGs du cluster "mycluster" sont listés + Then ce SG est inclus dans le résultat + + Scenario: SG d'un autre cluster + Given un SG avec la description "Security Group for Service LoadBalancer in cluster othercluster" + When les SGs du cluster "mycluster" sont listés + Then ce SG est exclu du résultat +``` + +#### US4.2 — Supprimer les Security Groups + +```gherkin +Feature: Suppression des Security Groups + Scenario: Suppression réussie + Given un SG appartenant au cluster "mycluster" + When la purge des SGs est déclenchée + Then le SG est supprimé via l'API Neutron + + Scenario: SG encore utilisé (HTTP 409) + Given une suppression de SG retourne HTTP 409 + When la purge tente de supprimer le SG + Then un warning est émis + And check_secgroups est true pour une vérification ultérieure +``` + +--- + +### Epic 5 — Gestion des Volumes Cinder + +#### US5.1 — Identifier les volumes d'un cluster + +```gherkin +Feature: Identification des volumes Cinder d'un cluster + Scenario: Volume appartenant au cluster sans marquage keep + Given un volume avec la métadonnée "cinder.csi.openstack.org/cluster" = "mycluster" + And la propriété "janitor.capi.azimuth-cloud.com/keep" est absente ou != "true" + When les volumes du cluster "mycluster" sont listés + Then ce volume est inclus dans le résultat + + Scenario: Volume marqué keep par l'utilisateur + Given un volume avec la métadonnée "cinder.csi.openstack.org/cluster" = "mycluster" + And la propriété "janitor.capi.azimuth-cloud.com/keep" = "true" + When les volumes du cluster "mycluster" sont listés + Then ce volume est exclu du résultat + + Scenario: Volume d'un autre cluster + Given un volume avec la métadonnée "cinder.csi.openstack.org/cluster" = "othercluster" + When les volumes du cluster "mycluster" sont listés + Then ce volume est exclu du résultat + + Scenario: Volume sans métadonnée CSI + Given un volume sans métadonnée "cinder.csi.openstack.org/cluster" + When les volumes du cluster "mycluster" sont listés + Then ce volume est exclu du résultat +``` + +#### US5.2 — Politique de suppression des volumes + +```gherkin +Feature: Politique de suppression des volumes + Scenario: Politique globale "delete" (défaut) + Given la variable d'environnement CAPI_JANITOR_DEFAULT_VOLUMES_POLICY non définie + When un cluster est supprimé sans annotation de volumes + Then les volumes du cluster sont supprimés + + Scenario: Politique globale "keep" + Given CAPI_JANITOR_DEFAULT_VOLUMES_POLICY = "keep" + When un cluster est supprimé sans annotation de volumes + Then les volumes du cluster sont conservés + + Scenario: Annotation "delete" sur le cluster (override keep global) + Given CAPI_JANITOR_DEFAULT_VOLUMES_POLICY = "keep" + And l'annotation "janitor.capi.stackhpc.com/volumes-policy" = "delete" sur l'OpenStackCluster + When le cluster est supprimé + Then les volumes du cluster sont supprimés + + Scenario: Annotation "keep" sur le cluster (override delete global) + Given CAPI_JANITOR_DEFAULT_VOLUMES_POLICY = "delete" + And l'annotation "janitor.capi.stackhpc.com/volumes-policy" = "keep" sur l'OpenStackCluster + When le cluster est supprimé + Then les volumes du cluster sont conservés +``` + +--- + +### Epic 6 — Gestion des Snapshots Cinder + +#### US6.1 — Identifier et supprimer les snapshots d'un cluster + +```gherkin +Feature: Snapshots Cinder d'un cluster + Scenario: Snapshot appartenant au cluster + Given un snapshot avec la métadonnée "cinder.csi.openstack.org/cluster" = "mycluster" + When les snapshots du cluster "mycluster" sont listés + Then ce snapshot est inclus dans le résultat + + Scenario: Snapshot d'un autre cluster + Given un snapshot avec la métadonnée "cinder.csi.openstack.org/cluster" = "othercluster" + When les snapshots du cluster "mycluster" sont listés + Then ce snapshot est exclu du résultat + + Scenario: Snapshots supprimés avant les volumes + Given des snapshots et des volumes appartenant au cluster "mycluster" + When la purge est déclenchée avec include_volumes = true + Then les snapshots sont supprimés en premier + And les volumes sont supprimés ensuite +``` + +--- + +### Epic 7 — Gestion des Application Credentials + +#### US7.1 — Supprimer l'Application Credential OpenStack + +```gherkin +Feature: Suppression de l'Application Credential + Scenario: Suppression autorisée (dernier finalizer) + Given l'annotation "janitor.capi.stackhpc.com/credential-policy" = "delete" sur le secret + And le finalizer de l'opérateur est le seul finalizer présent + When la purge des ressources OpenStack est terminée + Then l'Application Credential est supprimé via l'API Identity + And le secret Kubernetes contenant clouds.yaml est supprimé + + Scenario: Autres finalizers encore présents + Given l'annotation "credential-policy" = "delete" sur le secret + And d'autres finalizers sont encore présents sur l'OpenStackCluster + When la purge est terminée + Then l'Application Credential n'est pas supprimé + And une FinalizerStillPresentError est levée pour déclencher un retry + + Scenario: Application Credential non supprimable (403) + Given l'Application Credential est restreint (pas d'unrestricted) + When la suppression de l'appcred est tentée + Then un warning est émis + And la suppression du secret Kubernetes procède quand même +``` + +--- + +### Epic 8 — Lifecycle Kubernetes (pattern Finalizer) + +#### US8.1 — Ajouter un finalizer à la création + +```gherkin +Feature: Ajout du finalizer janitor sur OpenStackCluster + Scenario: Cluster sans deletionTimestamp et sans finalizer janitor + Given un OpenStackCluster sans deletionTimestamp + And sans finalizer "janitor.capi.stackhpc.com" + When un événement est reçu pour ce cluster + Then le finalizer "janitor.capi.stackhpc.com" est ajouté via patch + And un log INFO confirme l'ajout + + Scenario: Cluster avec finalizer déjà présent + Given un OpenStackCluster sans deletionTimestamp + And avec le finalizer "janitor.capi.stackhpc.com" déjà présent + When un événement est reçu + Then aucun patch n'est effectué +``` + +#### US8.2 — Nom du cluster depuis le label ou metadata.name + +```gherkin +Feature: Résolution du nom du cluster + Scenario: Label cluster.x-k8s.io/cluster-name présent + Given un OpenStackCluster avec le label "cluster.x-k8s.io/cluster-name" = "myapp" + And metadata.name = "myapp-openstack" + When l'opérateur résout le nom du cluster pour le nettoyage + Then le nom "myapp" est utilisé + + Scenario: Label absent + Given un OpenStackCluster sans label "cluster.x-k8s.io/cluster-name" + And metadata.name = "mycluster" + When l'opérateur résout le nom du cluster + Then le nom "mycluster" est utilisé +``` + +#### US8.3 — Supprimer le finalizer après nettoyage réussi + +```gherkin +Feature: Suppression du finalizer après purge + Scenario: Purge réussie + Given un OpenStackCluster en cours de suppression + And toutes les ressources OpenStack ont été supprimées + When la purge est terminée sans erreur + Then le finalizer "janitor.capi.stackhpc.com" est retiré via patch + And un log INFO confirme la suppression du finalizer + + Scenario: Finalizer absent au moment de la suppression + Given un OpenStackCluster avec deletionTimestamp + And sans finalizer "janitor.capi.stackhpc.com" + When un événement est reçu + Then aucune purge n'est déclenchée + And un log INFO indique que le finalizer est absent +``` + +#### US8.4 — Mécanisme de retry via annotation + +```gherkin +Feature: Retry via annotation aléatoire + Scenario: Erreur temporaire lors de la purge + Given une purge qui échoue avec une ResourcesStillPresentError + When l'opérateur gère l'erreur + Then après un délai de backoff (5s pour ResourcesStillPresent) + And une annotation aléatoire "janitor.capi.stackhpc.com/retry" est posée sur l'OpenStackCluster + And un nouvel événement est déclenché pour rejouer la purge + + Scenario: Erreur inconnue lors de la purge + Given une purge qui échoue avec une exception non classifiée + When l'opérateur gère l'erreur + Then le délai est CAPI_JANITOR_RETRY_DEFAULT_DELAY (défaut 60s) + And l'exception est loguée avec stack trace + + Scenario: Ressource supprimée entre l'erreur et le retry + Given l'OpenStackCluster est supprimé pendant le backoff + When l'opérateur tente d'annoter la ressource + Then l'ApiError 404 est ignorée +``` + +--- + +### Epic 9 — Configuration de l'opérateur + +#### US9.1 — Configuration via variables d'environnement + +```gherkin +Feature: Configuration via variables d'environnement + Scenario: Politique volumes par défaut configurée + Given CAPI_JANITOR_DEFAULT_VOLUMES_POLICY = "keep" + When l'opérateur démarre + Then la politique par défaut pour tous les clusters est "keep" + + Scenario: Délai de retry configurable + Given CAPI_JANITOR_RETRY_DEFAULT_DELAY = "120" + When une erreur non classifiée se produit + Then le délai de retry est 120 secondes +``` + +--- + +### Epic 10 — Packaging et déploiement + +#### US10.1 — Image Docker + +```gherkin +Feature: Image Docker de l'opérateur + Scenario: Build et push de l'image + Given le code source de l'opérateur Go + When le workflow GitHub Actions build-push-artifacts est déclenché + Then une image est publiée sur ghcr.io/azimuth-cloud/cluster-api-janitor-openstack + And l'image est taguée avec la version du chart + + Scenario: Sécurité de l'image + Given l'image Docker + Then le processus tourne en tant que non-root + And le système de fichiers racine est en lecture seule + And toutes les capabilities Linux sont droppées +``` + +#### US10.2 — Helm chart + +```gherkin +Feature: Déploiement via Helm chart + Scenario: Installation avec les valeurs par défaut + Given le Helm chart cluster-api-janitor-openstack + When helm install est exécuté + Then un Deployment, ServiceAccount, ClusterRole et ClusterRoleBinding sont créés + And la politique volumes par défaut est "delete" + + Scenario: Override de la politique volumes + Given helm install avec --set defaultVolumesPolicy=keep + When le chart est déployé + Then la variable CAPI_JANITOR_DEFAULT_VOLUMES_POLICY = "keep" est injectée dans le pod +``` + +--- + +### Epic 11 — Observabilité (nouvelle fonctionnalité) + +#### US11.1 — Métriques Prometheus + +```gherkin +Feature: Métriques Prometheus + Scenario: Comptage des purges réussies + Given une purge de cluster réussie + When les métriques sont exposées sur /metrics + Then le compteur "janitor_purge_total{status=success}" est incrémenté + + Scenario: Comptage des ressources supprimées par type + Given une purge ayant supprimé 3 FIPs, 2 LBs et 1 SG + When les métriques sont exposées + Then les jauges correspondantes reflètent les suppressions + + Scenario: Durée de purge + Given une purge terminée + When les métriques sont exposées + Then un histogramme "janitor_purge_duration_seconds" contient la durée de la purge +``` + +#### US11.2 — Status conditions sur OpenStackCluster + +```gherkin +Feature: Status conditions sur OpenStackCluster + Scenario: Purge en cours + Given une purge démarrée pour le cluster "mycluster" + When la purge est en cours + Then une condition "JanitorCleanupComplete" avec status "False" et reason "CleanupInProgress" est posée + + Scenario: Purge terminée avec succès + Given une purge réussie + When le finalizer est retiré + Then la condition "JanitorCleanupComplete" passe à status "True" + + Scenario: Purge en erreur + Given une purge échouant avec une erreur + When l'erreur est gérée + Then la condition "JanitorCleanupComplete" a status "False" et reason "CleanupFailed" avec un message d'erreur +``` + +--- + +### Epic 12 — Robustesse et extensibilité (nouvelles fonctionnalités) + +#### US12.1 — Timeout configurable pour les appels OpenStack + +```gherkin +Feature: Timeout HTTP configurable + Scenario: Appel OpenStack qui dépasse le timeout + Given CAPI_JANITOR_OPENSTACK_TIMEOUT = "30" + When un appel API OpenStack dépasse 30 secondes + Then le timeout est déclenché + And une erreur temporaire est loguée pour retry +``` + +#### US12.2 — Support des services Cinder avec alias + +```gherkin +Feature: Détection du service Cinder avec alias + Scenario: Catalogue avec "volumev3" + Given un catalogue OpenStack avec le service type "volumev3" + When l'opérateur cherche le client Cinder + Then le client "volumev3" est utilisé + + Scenario: Catalogue avec "block-storage" uniquement + Given un catalogue OpenStack sans "volumev3" mais avec "block-storage" + When l'opérateur cherche le client Cinder + Then le client "block-storage" est utilisé + + Scenario: Catalogue sans service Cinder + Given un catalogue sans "volumev3" ni "block-storage" + When l'opérateur cherche le client Cinder + Then une CatalogError est levée avec le message approprié +``` + +--- + +## Actions + +1. [x] Réaliser un audit du code +2. [ ] Écrire la feuille de route agile (ce document) +3. [ ] Scaffolding du projet Go avec kubebuilder (`/kubebuilder`) +4. [ ] Écrire les tests Go (TDD) pour chaque user story +5. [ ] Implémenter les fonctionnalités en Go +6. [ ] Migrer le Helm chart pour l'image Go +7. [ ] Implémenter les epics 11 (observabilité) et 12 (robustesse) + +## Ordre d'implémentation suggéré + +``` +Epic 1 (Auth) → Epic 2 (FIPs) → Epic 3 (LBs + PR #261) +→ Epic 4 (SGs) → Epic 5 (Volumes) → Epic 6 (Snapshots) +→ Epic 7 (AppCreds) → Epic 8 (Lifecycle K8s) → Epic 9 (Config) +→ Epic 10 (Packaging) → Epic 11 (Observabilité) → Epic 12 (Robustesse) +``` diff --git a/internal/openstack/cloud.go b/internal/openstack/cloud.go index e08bd87..f778fc9 100644 --- a/internal/openstack/cloud.go +++ b/internal/openstack/cloud.go @@ -6,6 +6,7 @@ import ( "context" "crypto/tls" "encoding/json" + "errors" "fmt" "io" "net/http" @@ -96,6 +97,12 @@ func (s *Session) IsAuthenticated() bool { return s.authenticated } // UserID returns the ID of the authenticated user. func (s *Session) UserID() string { return s.userID } +// HasEndpoint reports whether the session has a discovered endpoint for the given service type. +func (s *Session) HasEndpoint(serviceType string) bool { + _, ok := s.endpoints[serviceType] + return ok +} + // Authenticate performs token authentication against Keystone and discovers // the service catalog, returning a ready-to-use Session. func Authenticate(ctx context.Context, cloudsYAML, cloudName, cacert string) (*Session, error) { @@ -172,7 +179,11 @@ func (s *Session) getToken(ctx context.Context, baseURL string, auth authBlock) } `json:"user"` } `json:"token"` } - return json.NewDecoder(resp.Body).Decode(&result) + if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { + return err + } + s.userID = result.Token.User.ID + return nil } func (s *Session) loadCatalog(ctx context.Context, baseURL, iface, region string) error { @@ -280,24 +291,16 @@ func (e *httpStatusError) StatusCode() int { return e.code } func isHTTP(err error, code int) bool { var he *httpStatusError - if errors_as(err, &he) { + if errors.As(err, &he) { return he.code == code } return false } -func errors_as(err error, target **httpStatusError) bool { - if he, ok := err.(*httpStatusError); ok { - *target = he - return true - } - return false -} - // isTransient returns true for HTTP 400 and 409 (conflict/bad-request = retry). func isTransient(err error) bool { var he *httpStatusError - if errors_as(err, &he) { + if errors.As(err, &he) { return he.code == http.StatusBadRequest || he.code == http.StatusConflict } return false diff --git a/internal/openstack/cloud_test.go b/internal/openstack/cloud_test.go new file mode 100644 index 0000000..608fb63 --- /dev/null +++ b/internal/openstack/cloud_test.go @@ -0,0 +1,631 @@ +package openstack_test + +import ( + "context" + "crypto/tls" + "crypto/x509" + "encoding/json" + "encoding/pem" + "fmt" + "net/http" + "net/http/httptest" + "testing" + + "github.com/azimuth-cloud/cluster-api-janitor-openstack/internal/openstack" +) + +// keystoneServer is a configurable mock Keystone server for unit tests. +type keystoneServer struct { + *httptest.Server + tokenStatus int + tokenUserID string + catalogStatus int + catalog map[string]any +} + +func newKeystoneServer(t *testing.T) *keystoneServer { + t.Helper() + ks := &keystoneServer{ + tokenStatus: http.StatusCreated, + tokenUserID: "test-user-id", + catalogStatus: http.StatusOK, + catalog: map[string]any{"catalog": []any{}}, + } + mux := http.NewServeMux() + mux.HandleFunc("/v3/auth/tokens", func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + w.WriteHeader(http.StatusMethodNotAllowed) + return + } + if ks.tokenStatus >= 400 { + w.WriteHeader(ks.tokenStatus) + return + } + w.Header().Set("X-Subject-Token", "test-token-abc123") + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(ks.tokenStatus) + _ = json.NewEncoder(w).Encode(map[string]any{ + "token": map[string]any{ + "user": map[string]any{"id": ks.tokenUserID}, + }, + }) + }) + mux.HandleFunc("/v3/auth/catalog", func(w http.ResponseWriter, r *http.Request) { + if ks.catalogStatus == http.StatusNotFound { + w.WriteHeader(http.StatusNotFound) + return + } + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(ks.catalogStatus) + _ = json.NewEncoder(w).Encode(ks.catalog) + }) + ks.Server = httptest.NewServer(mux) + t.Cleanup(ks.Close) + return ks +} + +// newTLSKeystoneServer creates a TLS-enabled mock Keystone server. +func newTLSKeystoneServer(t *testing.T) *keystoneServer { + t.Helper() + ks := &keystoneServer{ + tokenStatus: http.StatusCreated, + tokenUserID: "test-user-id", + catalogStatus: http.StatusOK, + catalog: map[string]any{"catalog": []any{}}, + } + mux := http.NewServeMux() + mux.HandleFunc("/v3/auth/tokens", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("X-Subject-Token", "test-token-tls") + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusCreated) + _ = json.NewEncoder(w).Encode(map[string]any{ + "token": map[string]any{ + "user": map[string]any{"id": ks.tokenUserID}, + }, + }) + }) + mux.HandleFunc("/v3/auth/catalog", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + _ = json.NewEncoder(w).Encode(ks.catalog) + }) + ks.Server = httptest.NewTLSServer(mux) + t.Cleanup(ks.Close) + return ks +} + +func buildCloudsYAML(authURL, authType string) string { + return fmt.Sprintf(` +clouds: + openstack: + auth_type: %s + auth: + auth_url: %s + application_credential_id: appcred-abc123 + application_credential_secret: super-secret + region_name: RegionOne + interface: public +`, authType, authURL) +} + +func buildCloudsYAMLWithInterface(authURL, iface string) string { + return fmt.Sprintf(` +clouds: + openstack: + auth_type: v3applicationcredential + auth: + auth_url: %s + application_credential_id: appcred-abc123 + application_credential_secret: super-secret + interface: %s +`, authURL, iface) +} + +func buildCloudsYAMLWithRegion(authURL, region string) string { + if region == "" { + return fmt.Sprintf(` +clouds: + openstack: + auth_type: v3applicationcredential + auth: + auth_url: %s + application_credential_id: appcred-abc123 + application_credential_secret: super-secret + interface: public +`, authURL) + } + return fmt.Sprintf(` +clouds: + openstack: + auth_type: v3applicationcredential + auth: + auth_url: %s + application_credential_id: appcred-abc123 + application_credential_secret: super-secret + region_name: %s + interface: public +`, authURL, region) +} + +// catalogWith builds a catalog payload with one service and given endpoints. +func catalogWith(entries ...map[string]any) map[string]any { + return map[string]any{"catalog": entries} +} + +func serviceEntry(svcType string, endpoints ...map[string]any) map[string]any { + return map[string]any{ + "type": svcType, + "endpoints": endpoints, + } +} + +func endpoint(iface, regionID, url string) map[string]any { + return map[string]any{ + "interface": iface, + "region_id": regionID, + "url": url, + } +} + +// ── US1.1: Authentification via Application Credential v3 ────────────────── + +// Scenario: Authentification réussie +// Given un clouds.yaml avec auth_type "v3applicationcredential" +// And un application_credential_id et application_credential_secret valides +// When l'opérateur initialise la connexion OpenStack +// Then un token X-Auth-Token est obtenu depuis Keystone +// And le catalogue de services est chargé +func TestAuthenticate_SuccessfulAuthentication(t *testing.T) { + ks := newKeystoneServer(t) + ks.catalog = catalogWith( + serviceEntry("compute", + endpoint("public", "RegionOne", "http://compute.example.com"), + ), + ) + + clouds := buildCloudsYAML(ks.URL, "v3applicationcredential") + session, err := openstack.Authenticate(context.Background(), clouds, "openstack", "") + + if err != nil { + t.Fatalf("expected no error, got: %v", err) + } + if !session.IsAuthenticated() { + t.Error("expected session to be authenticated") + } + if session.UserID() != "test-user-id" { + t.Errorf("expected userID %q, got %q", "test-user-id", session.UserID()) + } +} + +// Scenario: Authentification avec un type non supporté +// Given un clouds.yaml avec auth_type "password" +// When l'opérateur tente de créer un client Cloud +// Then une erreur UnsupportedAuthTypeError est levée +func TestAuthenticate_UnsupportedAuthType(t *testing.T) { + ks := newKeystoneServer(t) + clouds := buildCloudsYAML(ks.URL, "password") + + _, err := openstack.Authenticate(context.Background(), clouds, "openstack", "") + + if err == nil { + t.Fatal("expected UnsupportedAuthTypeError, got nil") + } + var target *openstack.UnsupportedAuthTypeError + if !errorAs(err, &target) { + t.Fatalf("expected *UnsupportedAuthTypeError, got %T: %v", err, err) + } + if target.AuthType != "password" { + t.Errorf("expected AuthType %q, got %q", "password", target.AuthType) + } +} + +// Scenario: Cloud manquant dans clouds.yaml +func TestAuthenticate_CloudNotFound(t *testing.T) { + ks := newKeystoneServer(t) + clouds := buildCloudsYAML(ks.URL, "v3applicationcredential") + + _, err := openstack.Authenticate(context.Background(), clouds, "nonexistent-cloud", "") + + if err == nil { + t.Fatal("expected error for missing cloud, got nil") + } +} + +// Scenario: clouds.yaml invalide (YAML malformé) +func TestAuthenticate_InvalidCloudsYAML(t *testing.T) { + _, err := openstack.Authenticate(context.Background(), "not: valid: yaml: :", "openstack", "") + + if err == nil { + t.Fatal("expected parse error, got nil") + } +} + +// ── US1.3: Gestion d'un credential révoqué ou invalide ───────────────────── + +// Scenario: Application credential supprimé avant la purge (token request → 404) +// Given un cluster en cours de suppression +// And l'application credential a déjà été supprimé +// When l'opérateur tente de s'authentifier +// Then is_authenticated retourne false +// And aucune erreur fatale n'est levée +func TestAuthenticate_DeletedApplicationCredential_Returns404(t *testing.T) { + ks := newKeystoneServer(t) + ks.tokenStatus = http.StatusNotFound + + clouds := buildCloudsYAML(ks.URL, "v3applicationcredential") + session, err := openstack.Authenticate(context.Background(), clouds, "openstack", "") + + if err != nil { + t.Fatalf("expected no error for deleted appcred (404), got: %v", err) + } + if session.IsAuthenticated() { + t.Error("expected session to be unauthenticated when appcred is deleted") + } +} + +// Scenario: Token request échoue avec une erreur non-404 +// When l'opérateur tente de s'authentifier +// Then l'erreur est propagée +func TestAuthenticate_TokenRequestFails_NonFatal(t *testing.T) { + for _, code := range []int{http.StatusUnauthorized, http.StatusForbidden, http.StatusInternalServerError} { + code := code + t.Run(fmt.Sprintf("HTTP_%d", code), func(t *testing.T) { + ks := newKeystoneServer(t) + ks.tokenStatus = code + + clouds := buildCloudsYAML(ks.URL, "v3applicationcredential") + _, err := openstack.Authenticate(context.Background(), clouds, "openstack", "") + + if err == nil { + t.Fatalf("expected error for HTTP %d, got nil", code) + } + }) + } +} + +// Scenario: Catalogue retourne 404 +// Given une URL Keystone valide mais le catalogue retourne 404 +// When l'opérateur charge le catalogue +// Then is_authenticated retourne false +// And aucune erreur fatale n'est levée +func TestAuthenticate_CatalogReturns404(t *testing.T) { + ks := newKeystoneServer(t) + ks.catalogStatus = http.StatusNotFound + + clouds := buildCloudsYAML(ks.URL, "v3applicationcredential") + session, err := openstack.Authenticate(context.Background(), clouds, "openstack", "") + + if err != nil { + t.Fatalf("expected no error when catalog returns 404, got: %v", err) + } + if session.IsAuthenticated() { + t.Error("expected session to be unauthenticated when catalog returns 404") + } +} + +// ── US1.2: Filtrage du catalogue par interface et région ──────────────────── + +// Scenario: Endpoint sélectionné selon l'interface configurée +// Given un catalogue avec des endpoints "public" et "internal" +// And l'interface configurée est "public" +// When le catalogue est chargé +// Then seuls les endpoints "public" sont retenus +func TestAuthenticate_FiltersByInterface_Public(t *testing.T) { + ks := newKeystoneServer(t) + ks.catalog = catalogWith( + serviceEntry("compute", + endpoint("public", "RegionOne", "http://compute-public.example.com"), + endpoint("internal", "RegionOne", "http://compute-internal.example.com"), + ), + serviceEntry("network", + endpoint("internal", "RegionOne", "http://network-internal.example.com"), + ), + ) + + clouds := buildCloudsYAMLWithInterface(ks.URL, "public") + session, err := openstack.Authenticate(context.Background(), clouds, "openstack", "") + + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !session.IsAuthenticated() { + t.Fatal("expected session to be authenticated") + } + // compute endpoint should exist (public) + if !session.HasEndpoint("compute") { + t.Error("expected compute endpoint to be present") + } + // network endpoint should NOT exist (only internal) + if session.HasEndpoint("network") { + t.Error("expected network endpoint to be absent (only internal available)") + } +} + +// Scenario: Endpoint sélectionné selon l'interface configurée (internal) +func TestAuthenticate_FiltersByInterface_Internal(t *testing.T) { + ks := newKeystoneServer(t) + ks.catalog = catalogWith( + serviceEntry("compute", + endpoint("public", "RegionOne", "http://compute-public.example.com"), + ), + serviceEntry("network", + endpoint("internal", "RegionOne", "http://network-internal.example.com"), + ), + ) + + clouds := buildCloudsYAMLWithInterface(ks.URL, "internal") + session, err := openstack.Authenticate(context.Background(), clouds, "openstack", "") + + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if session.HasEndpoint("compute") { + t.Error("expected compute endpoint to be absent (only public available)") + } + if !session.HasEndpoint("network") { + t.Error("expected network endpoint to be present (internal)") + } +} + +// Scenario: Endpoint sélectionné selon la région configurée +// Given un catalogue avec des endpoints pour "RegionOne" et "RegionTwo" +// And la région configurée est "RegionOne" +// When le catalogue est chargé +// Then seuls les endpoints de "RegionOne" sont retenus +func TestAuthenticate_FiltersByRegion(t *testing.T) { + ks := newKeystoneServer(t) + ks.catalog = catalogWith( + serviceEntry("compute", + endpoint("public", "RegionOne", "http://compute-region1.example.com"), + ), + serviceEntry("network", + endpoint("public", "RegionTwo", "http://network-region2.example.com"), + ), + ) + + clouds := buildCloudsYAMLWithRegion(ks.URL, "RegionOne") + session, err := openstack.Authenticate(context.Background(), clouds, "openstack", "") + + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !session.HasEndpoint("compute") { + t.Error("expected compute endpoint (RegionOne) to be present") + } + if session.HasEndpoint("network") { + t.Error("expected network endpoint (RegionTwo) to be absent when region is RegionOne") + } +} + +// Scenario: Aucune région configurée +// Given un catalogue avec des endpoints dans plusieurs régions +// And aucune région n'est configurée +// When le catalogue est chargé +// Then le premier endpoint correspondant à l'interface est retenu pour chaque service +func TestAuthenticate_NoRegionFilter_AcceptsAllRegions(t *testing.T) { + ks := newKeystoneServer(t) + ks.catalog = catalogWith( + serviceEntry("compute", + endpoint("public", "RegionOne", "http://compute-region1.example.com"), + ), + serviceEntry("network", + endpoint("public", "RegionTwo", "http://network-region2.example.com"), + ), + ) + + clouds := buildCloudsYAMLWithRegion(ks.URL, "") // no region + session, err := openstack.Authenticate(context.Background(), clouds, "openstack", "") + + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !session.HasEndpoint("compute") { + t.Error("expected compute endpoint to be present when no region filter") + } + if !session.HasEndpoint("network") { + t.Error("expected network endpoint to be present when no region filter") + } +} + +// Scenario: Catalogue avec plusieurs services +func TestAuthenticate_MultipleServicesInCatalog(t *testing.T) { + ks := newKeystoneServer(t) + ks.catalog = catalogWith( + serviceEntry("compute", + endpoint("public", "RegionOne", "http://compute.example.com"), + ), + serviceEntry("network", + endpoint("public", "RegionOne", "http://network.example.com"), + ), + serviceEntry("identity", + endpoint("public", "RegionOne", "http://identity.example.com"), + ), + ) + + clouds := buildCloudsYAML(ks.URL, "v3applicationcredential") + session, err := openstack.Authenticate(context.Background(), clouds, "openstack", "") + + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + for _, svc := range []string{"compute", "network", "identity"} { + if !session.HasEndpoint(svc) { + t.Errorf("expected endpoint %q to be present", svc) + } + } +} + +// Scenario: Catalogue vide → non authentifié +func TestAuthenticate_EmptyCatalog_NotAuthenticated(t *testing.T) { + ks := newKeystoneServer(t) + ks.catalog = catalogWith() // no services + + clouds := buildCloudsYAML(ks.URL, "v3applicationcredential") + session, err := openstack.Authenticate(context.Background(), clouds, "openstack", "") + + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if session.IsAuthenticated() { + t.Error("expected session to be unauthenticated with empty catalog") + } +} + +// ── US1.4: Support des certificats CA personnalisés ───────────────────────── + +// Scenario: CA fourni dans le secret Kubernetes +// Given un secret Kubernetes contenant une entrée "cacert" +// When l'opérateur initialise le transport TLS +// Then le CA est chargé dans le contexte SSL +// And les appels HTTPS vers OpenStack utilisent ce CA pour la vérification +func TestAuthenticate_WithCustomCACert(t *testing.T) { + ks := newTLSKeystoneServer(t) + ks.catalog = catalogWith( + serviceEntry("compute", + endpoint("public", "RegionOne", "http://compute.example.com"), + ), + ) + + // Extract the server's self-signed certificate as PEM. + tlsConfig := ks.Server.TLS + rawCert := tlsConfig.Certificates[0].Certificate[0] + x509Cert, err := x509.ParseCertificate(rawCert) + if err != nil { + t.Fatalf("parsing server cert: %v", err) + } + certPEM := string(pem.EncodeToMemory(&pem.Block{ + Type: "CERTIFICATE", + Bytes: x509Cert.Raw, + })) + + clouds := buildCloudsYAML(ks.URL, "v3applicationcredential") + session, err := openstack.Authenticate(context.Background(), clouds, "openstack", certPEM) + + if err != nil { + t.Fatalf("expected no error with custom CA, got: %v", err) + } + if !session.IsAuthenticated() { + t.Error("expected authenticated session with valid custom CA cert") + } +} + +// Scenario: Pas de CA fourni → CA système utilisé (connexion HTTP doit fonctionner) +func TestAuthenticate_NoCACert_HTTPServerWorks(t *testing.T) { + ks := newKeystoneServer(t) // plain HTTP + ks.catalog = catalogWith( + serviceEntry("compute", + endpoint("public", "RegionOne", "http://compute.example.com"), + ), + ) + + clouds := buildCloudsYAML(ks.URL, "v3applicationcredential") + session, err := openstack.Authenticate(context.Background(), clouds, "openstack", "") + + if err != nil { + t.Fatalf("unexpected error without CA cert: %v", err) + } + if !session.IsAuthenticated() { + t.Error("expected authenticated session") + } +} + +// Scenario: CA fourni mais invalide → TLS doit échouer +func TestAuthenticate_InvalidCACert_TLSFails(t *testing.T) { + ks := newTLSKeystoneServer(t) + + clouds := buildCloudsYAML(ks.URL, "v3applicationcredential") + // Pass a wrong/empty CA cert → TLS verification should fail. + _, err := openstack.Authenticate(context.Background(), clouds, "openstack", "") + + if err == nil { + t.Fatal("expected TLS error without correct CA cert, got nil") + } +} + +// ── AppCredentialID ───────────────────────────────────────────────────────── + +// Scenario: Extraction de l'ID d'application credential depuis clouds.yaml +func TestAppCredentialID_Found(t *testing.T) { + clouds := ` +clouds: + mycloud: + auth_type: v3applicationcredential + auth: + auth_url: https://keystone.example.com/v3 + application_credential_id: my-appcred-id-xyz + application_credential_secret: secret123 +` + id, err := openstack.AppCredentialID(clouds, "mycloud") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if id != "my-appcred-id-xyz" { + t.Errorf("expected %q, got %q", "my-appcred-id-xyz", id) + } +} + +// Scenario: Cloud absent → erreur +func TestAppCredentialID_CloudNotFound(t *testing.T) { + clouds := ` +clouds: + openstack: + auth_type: v3applicationcredential + auth: + auth_url: https://keystone.example.com/v3 + application_credential_id: some-id + application_credential_secret: secret +` + _, err := openstack.AppCredentialID(clouds, "nonexistent") + if err == nil { + t.Fatal("expected error for missing cloud, got nil") + } +} + +// ── Error types ───────────────────────────────────────────────────────────── + +func TestAuthenticationError_Message(t *testing.T) { + err := &openstack.AuthenticationError{UserID: "user-abc"} + expected := "failed to authenticate as user: user-abc" + if err.Error() != expected { + t.Errorf("expected %q, got %q", expected, err.Error()) + } +} + +func TestUnsupportedAuthTypeError_Message(t *testing.T) { + err := &openstack.UnsupportedAuthTypeError{AuthType: "password"} + expected := "unsupported authentication type: password" + if err.Error() != expected { + t.Errorf("expected %q, got %q", expected, err.Error()) + } +} + +func TestCatalogError_Message(t *testing.T) { + err := &openstack.CatalogError{ServiceType: "volumev3"} + expected := "service type volumev3 not found in OpenStack service catalog" + if err.Error() != expected { + t.Errorf("expected %q, got %q", expected, err.Error()) + } +} + +// ── TLS verification ──────────────────────────────────────────────────────── + +// certPoolFromPEM builds an x509.CertPool from a PEM string (helper for assertions). +func certPoolFromPEM(pemData string) *x509.CertPool { + pool := x509.NewCertPool() + pool.AppendCertsFromPEM([]byte(pemData)) + return pool +} + +// tlsFromServer extracts TLS config from a test server (for CA assertions). +func tlsFromServer(s *httptest.Server) *tls.Config { + return s.TLS +} + +// errorAs is a typed helper to avoid importing errors package in test helpers. +func errorAs[T error](err error, target *T) bool { + if t, ok := err.(T); ok { + *target = t + return true + } + return false +} From 537c1157d84a34221ad46855b8e6c71aacbfcf87 Mon Sep 17 00:00:00 2001 From: Mathieu Grzybek Date: Tue, 7 Jul 2026 18:35:19 +0200 Subject: [PATCH 03/27] =?UTF-8?q?feat(openstack):=20Epic=202=20=E2=80=94?= =?UTF-8?q?=20Floating=20IP=20cleanup=20(TDD)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - resources_test.go (new): 13 tests covering US2.1 (filtering by cluster, description, multi-FIP) and US2.2 (successful deletion, transient 400/409 errors absorbed with verification, 500 error propagated, persistent FIPs → error, empty list → no verification, missing network endpoint → CatalogError) - resources.go — DeleteFloatingIPs: moved `deleted = true` before the DELETE call so verification always triggers on transient errors; added explicit log for those errors Signed-off-by: Mathieu Grzybek --- internal/openstack/resources.go | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/internal/openstack/resources.go b/internal/openstack/resources.go index be3916f..4afbaee 100644 --- a/internal/openstack/resources.go +++ b/internal/openstack/resources.go @@ -32,11 +32,17 @@ func (s *Session) DeleteFloatingIPs(ctx context.Context, logger logr.Logger, clu deleted := false for _, f := range list { if strings.HasPrefix(f.Description, prefix) && strings.HasSuffix(f.Description, suffix) { + // Mark found before attempting deletion so verification always runs, + // even when the delete returns a transient error. + deleted = true logger.Info("deleting floating IP", "id", f.ID) - if err := s.doDelete(ctx, networkURL+"/v2.0/floatingips/"+f.ID); err != nil && !isTransient(err) { - return err + if err := s.doDelete(ctx, networkURL+"/v2.0/floatingips/"+f.ID); err != nil { + if isTransient(err) { + logger.Info("transient error deleting floating IP, will verify", "id", f.ID, "error", err) + } else { + return err + } } - deleted = true } } if deleted { From 5c6bd741b39f3262e406d78e286344c72dfe7f2d Mon Sep 17 00:00:00 2001 From: Mathieu Grzybek Date: Tue, 7 Jul 2026 18:49:48 +0200 Subject: [PATCH 04/27] =?UTF-8?q?feat(openstack):=20Epic=203=20=E2=80=94?= =?UTF-8?q?=20Load=20Balancer=20cleanup=20(TDD)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - resources_test.go: 9 tests covering US3.1 (kube_service__ prefix filtering), US3.2 (HTTP error on list → log + return nil, no exception propagated), and US3.3 (cascade=true, successful deletion, transient 400/409 absorbed with verification, 500 propagated, persistent LBs → error, empty list → no verification, missing load-balancer endpoint → skip) - resources.go — DeleteLoadBalancers: moved deleted = true before DELETE call and added explicit warning log for transient errors (consistent with FIP cleanup pattern) Signed-off-by: Mathieu Grzybek --- internal/openstack/resources.go | 10 +- internal/openstack/resources_test.go | 799 +++++++++++++++++++++++++++ 2 files changed, 806 insertions(+), 3 deletions(-) create mode 100644 internal/openstack/resources_test.go diff --git a/internal/openstack/resources.go b/internal/openstack/resources.go index 4afbaee..5b058c2 100644 --- a/internal/openstack/resources.go +++ b/internal/openstack/resources.go @@ -82,12 +82,16 @@ func (s *Session) DeleteLoadBalancers(ctx context.Context, logger logr.Logger, c deleted := false for _, l := range list { if strings.HasPrefix(l.Name, kubePrefix) { + deleted = true logger.Info("deleting load balancer", "id", l.ID, "name", l.Name) target := lbURL + "/v2/lbaas/loadbalancers/" + l.ID + "?cascade=true" - if err := s.doDelete(ctx, target); err != nil && !isTransient(err) { - return err + if err := s.doDelete(ctx, target); err != nil { + if isTransient(err) { + logger.Info("transient error deleting load balancer, will verify", "id", l.ID, "error", err) + } else { + return err + } } - deleted = true } } if deleted { diff --git a/internal/openstack/resources_test.go b/internal/openstack/resources_test.go new file mode 100644 index 0000000..48e2b87 --- /dev/null +++ b/internal/openstack/resources_test.go @@ -0,0 +1,799 @@ +package openstack_test + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" + + "github.com/go-logr/logr" + + "github.com/azimuth-cloud/cluster-api-janitor-openstack/internal/openstack" +) + +// ── Mock server ─────────────────────────────────────────────────────────────── + +// fipRecord represents a Neutron floating IP in list responses. +type fipRecord struct { + ID string `json:"id"` + Description string `json:"description"` +} + +// networkTestServer is a mock OpenStack server that handles Keystone auth + +// a Neutron-like API on the same HTTP test server. The catalog advertises the +// server's own URL as the "network" endpoint, so Sessions authenticate and +// resolve resource URLs against this single server. +type networkTestServer struct { + *httptest.Server + mu sync.Mutex + fipLists [][]fipRecord // sequence of list responses; last entry is reused + fipGetCount int // total number of GET /v2.0/floatingips calls + // Per-FIP DELETE response status (default 204 NoContent). + deleteStatus map[string]int + // IDs deleted in call order. + deletedFIPs []string +} + +func newNetworkTestServer(t *testing.T) *networkTestServer { + t.Helper() + srv := &networkTestServer{ + deleteStatus: make(map[string]int), + } + mux := http.NewServeMux() + + // Keystone: token endpoint + mux.HandleFunc("/v3/auth/tokens", func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + w.WriteHeader(http.StatusMethodNotAllowed) + return + } + w.Header().Set("X-Subject-Token", "test-network-token") + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusCreated) + _ = json.NewEncoder(w).Encode(map[string]any{ + "token": map[string]any{"user": map[string]any{"id": "test-user"}}, + }) + }) + + // Keystone: service catalog — self-referential: "network" endpoint = this server + mux.HandleFunc("/v3/auth/catalog", func(w http.ResponseWriter, r *http.Request) { + selfURL := "http://" + r.Host + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{ + "catalog": []any{ + map[string]any{ + "type": "network", + "endpoints": []any{ + map[string]any{ + "interface": "public", + "region_id": "RegionOne", + "url": selfURL, + }, + }, + }, + }, + }) + }) + + // Neutron: list floating IPs (exact path — no trailing slash to avoid redirect) + mux.HandleFunc("/v2.0/floatingips", func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + w.WriteHeader(http.StatusMethodNotAllowed) + return + } + srv.mu.Lock() + srv.fipGetCount++ + idx := srv.fipGetCount - 1 + var list []fipRecord + if len(srv.fipLists) > 0 { + if idx < len(srv.fipLists) { + list = srv.fipLists[idx] + } else { + list = srv.fipLists[len(srv.fipLists)-1] + } + } + srv.mu.Unlock() + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{"floatingips": list}) + }) + + // Neutron: delete floating IP by ID (subtree pattern handles /v2.0/floatingips/{id}) + mux.HandleFunc("/v2.0/floatingips/", func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodDelete { + w.WriteHeader(http.StatusMethodNotAllowed) + return + } + id := strings.TrimPrefix(r.URL.Path, "/v2.0/floatingips/") + srv.mu.Lock() + srv.deletedFIPs = append(srv.deletedFIPs, id) + status, ok := srv.deleteStatus[id] + if !ok { + status = http.StatusNoContent + } + srv.mu.Unlock() + w.WriteHeader(status) + }) + + srv.Server = httptest.NewServer(mux) + t.Cleanup(srv.Server.Close) + return srv +} + +// authenticate creates an authenticated Session against this server. +func (srv *networkTestServer) authenticate(t *testing.T) *openstack.Session { + t.Helper() + cloudsYAML := fmt.Sprintf(` +clouds: + openstack: + auth_type: v3applicationcredential + auth: + auth_url: %s + application_credential_id: test-id + application_credential_secret: test-secret + interface: public + region_name: RegionOne +`, srv.URL) + session, err := openstack.Authenticate(context.Background(), cloudsYAML, "openstack", "") + if err != nil { + t.Fatalf("authenticate: %v", err) + } + if !session.IsAuthenticated() { + t.Fatal("expected authenticated session with network endpoint") + } + return session +} + +// fipDesc returns the description that OCCM writes for a service FIP. +func fipDesc(cluster string) string { + return fmt.Sprintf("Floating IP for Kubernetes external service from cluster %s", cluster) +} + +// ── LB mock server ─────────────────────────────────────────────────────────── + +type lbRecord struct { + ID string `json:"id"` + Name string `json:"name"` +} + +// lbTestServer is a mock OpenStack server that handles Keystone auth + +// an Octavia-like API on the same HTTP test server. The catalog advertises the +// server's own URL as the "load-balancer" endpoint. +type lbTestServer struct { + *httptest.Server + mu sync.Mutex + lbLists [][]lbRecord + lbGetCount int + listStatusOverride int // if non-zero, all GET /v2/lbaas/loadbalancers return this status + deleteStatus map[string]int + deletedLBs []string + deleteCascade []string // cascade query param value per DELETE call +} + +func newLBTestServer(t *testing.T) *lbTestServer { + t.Helper() + srv := &lbTestServer{ + deleteStatus: make(map[string]int), + } + mux := http.NewServeMux() + + mux.HandleFunc("/v3/auth/tokens", func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + w.WriteHeader(http.StatusMethodNotAllowed) + return + } + w.Header().Set("X-Subject-Token", "test-lb-token") + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusCreated) + _ = json.NewEncoder(w).Encode(map[string]any{ + "token": map[string]any{"user": map[string]any{"id": "test-user"}}, + }) + }) + + // Self-referential catalog: "load-balancer" endpoint = this server. + mux.HandleFunc("/v3/auth/catalog", func(w http.ResponseWriter, r *http.Request) { + selfURL := "http://" + r.Host + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{ + "catalog": []any{ + map[string]any{ + "type": "load-balancer", + "endpoints": []any{ + map[string]any{ + "interface": "public", + "region_id": "RegionOne", + "url": selfURL, + }, + }, + }, + }, + }) + }) + + mux.HandleFunc("/v2/lbaas/loadbalancers", func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + w.WriteHeader(http.StatusMethodNotAllowed) + return + } + srv.mu.Lock() + override := srv.listStatusOverride + srv.lbGetCount++ + idx := srv.lbGetCount - 1 + var list []lbRecord + if len(srv.lbLists) > 0 { + if idx < len(srv.lbLists) { + list = srv.lbLists[idx] + } else { + list = srv.lbLists[len(srv.lbLists)-1] + } + } + srv.mu.Unlock() + if override != 0 { + w.WriteHeader(override) + return + } + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{"loadbalancers": list}) + }) + + mux.HandleFunc("/v2/lbaas/loadbalancers/", func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodDelete { + w.WriteHeader(http.StatusMethodNotAllowed) + return + } + id := strings.TrimPrefix(r.URL.Path, "/v2/lbaas/loadbalancers/") + cascade := r.URL.Query().Get("cascade") + srv.mu.Lock() + srv.deletedLBs = append(srv.deletedLBs, id) + srv.deleteCascade = append(srv.deleteCascade, cascade) + status, ok := srv.deleteStatus[id] + if !ok { + status = http.StatusNoContent + } + srv.mu.Unlock() + w.WriteHeader(status) + }) + + srv.Server = httptest.NewServer(mux) + t.Cleanup(srv.Server.Close) + return srv +} + +func (srv *lbTestServer) authenticate(t *testing.T) *openstack.Session { + t.Helper() + cloudsYAML := fmt.Sprintf(` +clouds: + openstack: + auth_type: v3applicationcredential + auth: + auth_url: %s + application_credential_id: test-id + application_credential_secret: test-secret + interface: public + region_name: RegionOne +`, srv.URL) + session, err := openstack.Authenticate(context.Background(), cloudsYAML, "openstack", "") + if err != nil { + t.Fatalf("authenticate: %v", err) + } + return session +} + +func lbKubeName(cluster, suffix string) string { + return fmt.Sprintf("kube_service_%s_%s", cluster, suffix) +} + +// ── US2.1: Identifier les Floating IPs d'un cluster ────────────────────────── + +// Scenario: FIP appartenant au cluster → incluse dans la suppression +// Scenario: FIP d'un autre cluster → exclue +// Scenario: FIP sans description Kubernetes → exclue +func TestDeleteFloatingIPs_Filtering(t *testing.T) { + tests := []struct { + name string + description string + cluster string + shouldDelete bool + }{ + { + name: "matching cluster", + description: fipDesc("mycluster"), + cluster: "mycluster", + shouldDelete: true, + }, + { + name: "different cluster", + description: fipDesc("othercluster"), + cluster: "mycluster", + shouldDelete: false, + }, + { + name: "non-kubernetes description", + description: "Some unrelated floating IP", + cluster: "mycluster", + shouldDelete: false, + }, + { + name: "empty description", + description: "", + cluster: "mycluster", + shouldDelete: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + srv := newNetworkTestServer(t) + fip := fipRecord{ID: "fip-001", Description: tt.description} + // List: FIP present before deletion; empty after (verification passes) + srv.fipLists = [][]fipRecord{{fip}, {}} + + session := srv.authenticate(t) + if err := session.DeleteFloatingIPs(context.Background(), logr.Discard(), tt.cluster); err != nil { + t.Fatalf("unexpected error: %v", err) + } + + srv.mu.Lock() + deleted := len(srv.deletedFIPs) > 0 + srv.mu.Unlock() + + if deleted != tt.shouldDelete { + t.Errorf("shouldDelete=%v but FIP was deleted=%v", tt.shouldDelete, deleted) + } + }) + } +} + +// Scenario: FIPs de plusieurs clusters → seules celles du bon cluster sont supprimées +func TestDeleteFloatingIPs_MultipleIPsPartialMatch(t *testing.T) { + srv := newNetworkTestServer(t) + fips := []fipRecord{ + {ID: "fip-001", Description: fipDesc("mycluster")}, // match + {ID: "fip-002", Description: fipDesc("othercluster")}, // no match + {ID: "fip-003", Description: "Some other description"}, // no match + {ID: "fip-004", Description: fipDesc("mycluster")}, // match + } + srv.fipLists = [][]fipRecord{fips, {}} + + session := srv.authenticate(t) + if err := session.DeleteFloatingIPs(context.Background(), logr.Discard(), "mycluster"); err != nil { + t.Fatalf("unexpected error: %v", err) + } + + srv.mu.Lock() + deleted := make(map[string]bool, len(srv.deletedFIPs)) + for _, id := range srv.deletedFIPs { + deleted[id] = true + } + srv.mu.Unlock() + + if !deleted["fip-001"] || !deleted["fip-004"] { + t.Errorf("expected fip-001 and fip-004 to be deleted, got: %v", srv.deletedFIPs) + } + if deleted["fip-002"] || deleted["fip-003"] { + t.Errorf("expected fip-002 and fip-003 to NOT be deleted, got: %v", srv.deletedFIPs) + } +} + +// ── US2.2: Supprimer les Floating IPs ──────────────────────────────────────── + +// Scenario: Suppression réussie +// Given une FIP appartenant au cluster "mycluster" +// When la purge des FIPs est déclenchée +// Then la FIP est supprimée via l'API Neutron +func TestDeleteFloatingIPs_SuccessfulDeletion(t *testing.T) { + srv := newNetworkTestServer(t) + fips := []fipRecord{ + {ID: "fip-001", Description: fipDesc("mycluster")}, + {ID: "fip-002", Description: fipDesc("mycluster")}, + } + srv.fipLists = [][]fipRecord{fips, {}} // first: FIPs present; verification: empty + + session := srv.authenticate(t) + if err := session.DeleteFloatingIPs(context.Background(), logr.Discard(), "mycluster"); err != nil { + t.Fatalf("unexpected error: %v", err) + } + + srv.mu.Lock() + deletedIDs := srv.deletedFIPs + srv.mu.Unlock() + + if len(deletedIDs) != 2 { + t.Errorf("expected 2 FIPs deleted, got %d: %v", len(deletedIDs), deletedIDs) + } + idSet := make(map[string]bool) + for _, id := range deletedIDs { + idSet[id] = true + } + for _, fip := range fips { + if !idSet[fip.ID] { + t.Errorf("expected FIP %s to be deleted", fip.ID) + } + } +} + +// Scenario: Erreur HTTP 400 lors de la suppression +// Then un warning est émis +// And la suppression continue pour les autres FIPs +// And vérification déclenchée (check_fips = true) +func TestDeleteFloatingIPs_TransientError400_ContinuesAndVerifies(t *testing.T) { + srv := newNetworkTestServer(t) + fips := []fipRecord{ + {ID: "fip-001", Description: fipDesc("mycluster")}, // returns HTTP 400 + {ID: "fip-002", Description: fipDesc("mycluster")}, // returns HTTP 204 + } + srv.fipLists = [][]fipRecord{fips, {}} // verification returns empty + srv.deleteStatus["fip-001"] = http.StatusBadRequest + + session := srv.authenticate(t) + err := session.DeleteFloatingIPs(context.Background(), logr.Discard(), "mycluster") + + // Transient error must NOT be propagated + if err != nil { + t.Fatalf("expected no error for transient HTTP 400, got: %v", err) + } + + // Both FIPs must have been attempted + srv.mu.Lock() + attempted := len(srv.deletedFIPs) + getCount := srv.fipGetCount + srv.mu.Unlock() + + if attempted != 2 { + t.Errorf("expected both FIPs to be attempted, got %d DELETE calls", attempted) + } + // Verification GET must have been triggered (deleted=true even on transient error) + if getCount < 2 { + t.Errorf("expected at least 2 GET calls (list + verification), got %d", getCount) + } +} + +// Scenario: Erreur HTTP 409 (Conflict) → même comportement que 400 +func TestDeleteFloatingIPs_TransientError409_Continues(t *testing.T) { + srv := newNetworkTestServer(t) + fips := []fipRecord{ + {ID: "fip-conflict", Description: fipDesc("mycluster")}, + } + srv.fipLists = [][]fipRecord{fips, {}} + srv.deleteStatus["fip-conflict"] = http.StatusConflict + + session := srv.authenticate(t) + if err := session.DeleteFloatingIPs(context.Background(), logr.Discard(), "mycluster"); err != nil { + t.Fatalf("expected no error for transient HTTP 409, got: %v", err) + } +} + +// Scenario: Erreur HTTP 500 → exception propagée +func TestDeleteFloatingIPs_HTTP500_PropagatesError(t *testing.T) { + srv := newNetworkTestServer(t) + fips := []fipRecord{ + {ID: "fip-server-error", Description: fipDesc("mycluster")}, + } + srv.fipLists = [][]fipRecord{fips} + srv.deleteStatus["fip-server-error"] = http.StatusInternalServerError + + session := srv.authenticate(t) + err := session.DeleteFloatingIPs(context.Background(), logr.Discard(), "mycluster") + + if err == nil { + t.Fatal("expected error for HTTP 500, got nil") + } +} + +// Scenario: FIPs toujours présentes après suppression → erreur retournée +// (le contrôleur réessaiera via l'annotation retry) +func TestDeleteFloatingIPs_StillPresentAfterDeletion_ReturnsError(t *testing.T) { + srv := newNetworkTestServer(t) + fip := fipRecord{ID: "fip-persistent", Description: fipDesc("mycluster")} + // Verification also returns the FIP — OpenStack has not deleted it yet + srv.fipLists = [][]fipRecord{{fip}, {fip}} + + session := srv.authenticate(t) + err := session.DeleteFloatingIPs(context.Background(), logr.Discard(), "mycluster") + + if err == nil { + t.Fatal("expected error when FIP persists after deletion") + } + if !strings.Contains(err.Error(), "mycluster") { + t.Errorf("expected error to mention cluster name, got: %v", err) + } +} + +// Scenario: Aucune FIP correspondante → pas de suppression, pas de vérification +func TestDeleteFloatingIPs_NothingToDelete_NoVerification(t *testing.T) { + srv := newNetworkTestServer(t) + srv.fipLists = [][]fipRecord{{}} // empty list + + session := srv.authenticate(t) + if err := session.DeleteFloatingIPs(context.Background(), logr.Discard(), "mycluster"); err != nil { + t.Fatalf("unexpected error with no FIPs: %v", err) + } + + srv.mu.Lock() + deletedCount := len(srv.deletedFIPs) + getCount := srv.fipGetCount + srv.mu.Unlock() + + if deletedCount != 0 { + t.Errorf("expected no DELETE calls, got %d: %v", deletedCount, srv.deletedFIPs) + } + // No verification GET should be triggered when nothing was found + if getCount != 1 { + t.Errorf("expected exactly 1 GET call (no verification), got %d", getCount) + } +} + +// Scenario: Pas d'endpoint "network" dans le catalogue → CatalogError +func TestDeleteFloatingIPs_NoNetworkEndpoint_ReturnsCatalogError(t *testing.T) { + // Use a Keystone server that only advertises a "compute" endpoint (no "network") + ks := newKeystoneServer(t) + ks.catalog = catalogWith( + serviceEntry("compute", + endpoint("public", "RegionOne", "http://compute.example.com"), + ), + ) + + clouds := buildCloudsYAML(ks.URL, "v3applicationcredential") + session, err := openstack.Authenticate(context.Background(), clouds, "openstack", "") + if err != nil { + t.Fatalf("unexpected authenticate error: %v", err) + } + + err = session.DeleteFloatingIPs(context.Background(), logr.Discard(), "mycluster") + + if err == nil { + t.Fatal("expected CatalogError when network endpoint is absent") + } + var target *openstack.CatalogError + if !errorAs(err, &target) { + t.Errorf("expected *CatalogError, got %T: %v", err, err) + } +} + +// ── Epic 3: Nettoyage des Load Balancers Octavia ────────────────────────────── + +// ── US3.1: Identifier les Load Balancers Kubernetes ────────────────────────── + +// Scenario: LB appartenant au cluster → inclus dans la suppression +// Scenario: LB d'un autre cluster → exclu +// Scenario: LB sans préfixe kube_service → exclu +func TestDeleteLoadBalancers_Filtering(t *testing.T) { + tests := []struct { + name string + lbName string + cluster string + shouldDelete bool + }{ + { + name: "matching cluster", + lbName: lbKubeName("mycluster", "api"), + cluster: "mycluster", + shouldDelete: true, + }, + { + name: "different cluster", + lbName: lbKubeName("othercluster", "api"), + cluster: "mycluster", + shouldDelete: false, + }, + { + name: "wrong prefix", + lbName: "fake_service_mycluster_api", + cluster: "mycluster", + shouldDelete: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + srv := newLBTestServer(t) + lb := lbRecord{ID: "lb-001", Name: tt.lbName} + srv.lbLists = [][]lbRecord{{lb}, {}} + + session := srv.authenticate(t) + if err := session.DeleteLoadBalancers(context.Background(), logr.Discard(), tt.cluster); err != nil { + t.Fatalf("unexpected error: %v", err) + } + + srv.mu.Lock() + deleted := len(srv.deletedLBs) > 0 + srv.mu.Unlock() + + if deleted != tt.shouldDelete { + t.Errorf("shouldDelete=%v but LB was deleted=%v", tt.shouldDelete, deleted) + } + }) + } +} + +// ── US3.2: Erreur HTTP lors du listing (PR #261) ────────────────────────────── + +// Scenario: Erreur HTTP lors du listing des LBs → log ERROR, pas d'exception +func TestDeleteLoadBalancers_ListError_LogsAndSkips(t *testing.T) { + srv := newLBTestServer(t) + srv.listStatusOverride = http.StatusInternalServerError + + session := srv.authenticate(t) + err := session.DeleteLoadBalancers(context.Background(), logr.Discard(), "mycluster") + + if err != nil { + t.Fatalf("expected nil when list returns HTTP 500, got: %v", err) + } + srv.mu.Lock() + deleted := len(srv.deletedLBs) + srv.mu.Unlock() + if deleted != 0 { + t.Errorf("expected no DELETE calls when list fails, got %d", deleted) + } +} + +// ── US3.3: Supprimer les Load Balancers en cascade ─────────────────────────── + +// Scenario: Suppression réussie avec cascade=true +func TestDeleteLoadBalancers_SuccessfulDeletion(t *testing.T) { + srv := newLBTestServer(t) + lbs := []lbRecord{ + {ID: "lb-001", Name: lbKubeName("mycluster", "svc1")}, + {ID: "lb-002", Name: lbKubeName("mycluster", "svc2")}, + } + srv.lbLists = [][]lbRecord{lbs, {}} + + session := srv.authenticate(t) + if err := session.DeleteLoadBalancers(context.Background(), logr.Discard(), "mycluster"); err != nil { + t.Fatalf("unexpected error: %v", err) + } + + srv.mu.Lock() + deletedIDs := srv.deletedLBs + srv.mu.Unlock() + + if len(deletedIDs) != 2 { + t.Errorf("expected 2 LBs deleted, got %d: %v", len(deletedIDs), deletedIDs) + } + idSet := make(map[string]bool) + for _, id := range deletedIDs { + idSet[id] = true + } + for _, lb := range lbs { + if !idSet[lb.ID] { + t.Errorf("expected LB %s to be deleted", lb.ID) + } + } +} + +// Scenario: DELETE émis avec cascade=true +func TestDeleteLoadBalancers_CascadeDelete(t *testing.T) { + srv := newLBTestServer(t) + srv.lbLists = [][]lbRecord{ + {{ID: "lb-cascade", Name: lbKubeName("mycluster", "svc")}}, + {}, + } + + session := srv.authenticate(t) + if err := session.DeleteLoadBalancers(context.Background(), logr.Discard(), "mycluster"); err != nil { + t.Fatalf("unexpected error: %v", err) + } + + srv.mu.Lock() + cascades := srv.deleteCascade + srv.mu.Unlock() + + if len(cascades) == 0 { + t.Fatal("expected a DELETE call, got none") + } + for i, c := range cascades { + if c != "true" { + t.Errorf("DELETE call %d: expected cascade=true, got %q", i, c) + } + } +} + +// Scenario: Erreur HTTP 400 lors de la suppression → warning, continue, vérification déclenchée +func TestDeleteLoadBalancers_TransientError400_ContinuesAndVerifies(t *testing.T) { + srv := newLBTestServer(t) + lbs := []lbRecord{ + {ID: "lb-001", Name: lbKubeName("mycluster", "svc1")}, // returns HTTP 400 + {ID: "lb-002", Name: lbKubeName("mycluster", "svc2")}, // returns HTTP 204 + } + srv.lbLists = [][]lbRecord{lbs, {}} + srv.deleteStatus["lb-001"] = http.StatusBadRequest + + session := srv.authenticate(t) + err := session.DeleteLoadBalancers(context.Background(), logr.Discard(), "mycluster") + + if err != nil { + t.Fatalf("expected no error for transient HTTP 400, got: %v", err) + } + + srv.mu.Lock() + attempted := len(srv.deletedLBs) + getCount := srv.lbGetCount + srv.mu.Unlock() + + if attempted != 2 { + t.Errorf("expected both LBs to be attempted, got %d DELETE calls", attempted) + } + if getCount < 2 { + t.Errorf("expected at least 2 GET calls (list + verification), got %d", getCount) + } +} + +// Scenario: Erreur HTTP 500 → exception propagée +func TestDeleteLoadBalancers_HTTP500_PropagatesError(t *testing.T) { + srv := newLBTestServer(t) + srv.lbLists = [][]lbRecord{ + {{ID: "lb-server-error", Name: lbKubeName("mycluster", "svc")}}, + } + srv.deleteStatus["lb-server-error"] = http.StatusInternalServerError + + session := srv.authenticate(t) + err := session.DeleteLoadBalancers(context.Background(), logr.Discard(), "mycluster") + + if err == nil { + t.Fatal("expected error for HTTP 500, got nil") + } +} + +// Scenario: LBs toujours présents après suppression → erreur retournée +func TestDeleteLoadBalancers_StillPresentAfterDeletion_ReturnsError(t *testing.T) { + srv := newLBTestServer(t) + lb := lbRecord{ID: "lb-persistent", Name: lbKubeName("mycluster", "svc")} + srv.lbLists = [][]lbRecord{{lb}, {lb}} + + session := srv.authenticate(t) + err := session.DeleteLoadBalancers(context.Background(), logr.Discard(), "mycluster") + + if err == nil { + t.Fatal("expected error when LB persists after deletion") + } + if !strings.Contains(err.Error(), "mycluster") { + t.Errorf("expected error to mention cluster name, got: %v", err) + } +} + +// Scenario: Aucun LB correspondant → pas de suppression, pas de vérification +func TestDeleteLoadBalancers_NothingToDelete_NoVerification(t *testing.T) { + srv := newLBTestServer(t) + srv.lbLists = [][]lbRecord{{}} + + session := srv.authenticate(t) + if err := session.DeleteLoadBalancers(context.Background(), logr.Discard(), "mycluster"); err != nil { + t.Fatalf("unexpected error with no LBs: %v", err) + } + + srv.mu.Lock() + deletedCount := len(srv.deletedLBs) + getCount := srv.lbGetCount + srv.mu.Unlock() + + if deletedCount != 0 { + t.Errorf("expected no DELETE calls, got %d: %v", deletedCount, srv.deletedLBs) + } + if getCount != 1 { + t.Errorf("expected exactly 1 GET call (no verification), got %d", getCount) + } +} + +// Scenario: Pas d'endpoint "load-balancer" dans le catalogue → retour nil (LBs ignorés) +func TestDeleteLoadBalancers_NoLoadBalancerEndpoint_Skips(t *testing.T) { + ks := newKeystoneServer(t) + ks.catalog = catalogWith( + serviceEntry("network", + endpoint("public", "RegionOne", "http://network.example.com"), + ), + ) + + clouds := buildCloudsYAML(ks.URL, "v3applicationcredential") + session, err := openstack.Authenticate(context.Background(), clouds, "openstack", "") + if err != nil { + t.Fatalf("unexpected authenticate error: %v", err) + } + + err = session.DeleteLoadBalancers(context.Background(), logr.Discard(), "mycluster") + if err != nil { + t.Fatalf("expected nil when load-balancer endpoint is absent, got: %v", err) + } +} From c887846b172414b4cfeff9e2561c4f717050bbd2 Mon Sep 17 00:00:00 2001 From: Mathieu Grzybek Date: Tue, 7 Jul 2026 18:56:12 +0200 Subject: [PATCH 05/27] =?UTF-8?q?feat(openstack):=20Epic=204=20=E2=80=94?= =?UTF-8?q?=20Security=20Group=20cleanup=20(TDD)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - resources_test.go: 6 tests covering US4.1 (description prefix/suffix filtering, different cluster, wrong prefix, unrelated description), US4.2 (successful deletion, transient HTTP 409 absorbed with verification, 500 propagated, persistent SG → error, nothing to delete → no verification) - resources.go — DeleteSecurityGroups: moved deleted = true before DELETE call; added explicit warning log for transient errors (consistent with FIP/LB cleanup pattern) Signed-off-by: Mathieu Grzybek --- internal/openstack/resources.go | 10 +- internal/openstack/resources_test.go | 317 +++++++++++++++++++++++++++ 2 files changed, 324 insertions(+), 3 deletions(-) diff --git a/internal/openstack/resources.go b/internal/openstack/resources.go index 5b058c2..93278f4 100644 --- a/internal/openstack/resources.go +++ b/internal/openstack/resources.go @@ -130,11 +130,15 @@ func (s *Session) DeleteSecurityGroups(ctx context.Context, logger logr.Logger, deleted := false for _, g := range list { if strings.HasPrefix(g.Description, "Security Group for") && strings.HasSuffix(g.Description, sgSuffix) { + deleted = true logger.Info("deleting security group", "id", g.ID) - if err := s.doDelete(ctx, networkURL+"/v2.0/security-groups/"+g.ID); err != nil && !isTransient(err) { - return err + if err := s.doDelete(ctx, networkURL+"/v2.0/security-groups/"+g.ID); err != nil { + if isTransient(err) { + logger.Info("transient error deleting security group, will verify", "id", g.ID, "error", err) + } else { + return err + } } - deleted = true } } if deleted { diff --git a/internal/openstack/resources_test.go b/internal/openstack/resources_test.go index 48e2b87..abc89fc 100644 --- a/internal/openstack/resources_test.go +++ b/internal/openstack/resources_test.go @@ -777,6 +777,140 @@ func TestDeleteLoadBalancers_NothingToDelete_NoVerification(t *testing.T) { } } +// ── SG mock server ──────────────────────────────────────────────────────────── + +type sgRecord struct { + ID string `json:"id"` + Description string `json:"description"` +} + +// sgTestServer is a mock OpenStack server that handles Keystone auth + +// a Neutron-like security group API. The catalog advertises the server's own +// URL as the "network" endpoint. +type sgTestServer struct { + *httptest.Server + mu sync.Mutex + sgLists [][]sgRecord + sgGetCount int + listStatusOverride int // if non-zero, all GETs return this status + deleteStatus map[string]int + deletedSGs []string +} + +func newSGTestServer(t *testing.T) *sgTestServer { + t.Helper() + srv := &sgTestServer{ + deleteStatus: make(map[string]int), + } + mux := http.NewServeMux() + + mux.HandleFunc("/v3/auth/tokens", func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + w.WriteHeader(http.StatusMethodNotAllowed) + return + } + w.Header().Set("X-Subject-Token", "test-sg-token") + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusCreated) + _ = json.NewEncoder(w).Encode(map[string]any{ + "token": map[string]any{"user": map[string]any{"id": "test-user"}}, + }) + }) + + // Self-referential catalog: "network" endpoint = this server. + mux.HandleFunc("/v3/auth/catalog", func(w http.ResponseWriter, r *http.Request) { + selfURL := "http://" + r.Host + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{ + "catalog": []any{ + map[string]any{ + "type": "network", + "endpoints": []any{ + map[string]any{ + "interface": "public", + "region_id": "RegionOne", + "url": selfURL, + }, + }, + }, + }, + }) + }) + + mux.HandleFunc("/v2.0/security-groups", func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + w.WriteHeader(http.StatusMethodNotAllowed) + return + } + srv.mu.Lock() + override := srv.listStatusOverride + srv.sgGetCount++ + idx := srv.sgGetCount - 1 + var list []sgRecord + if len(srv.sgLists) > 0 { + if idx < len(srv.sgLists) { + list = srv.sgLists[idx] + } else { + list = srv.sgLists[len(srv.sgLists)-1] + } + } + srv.mu.Unlock() + if override != 0 { + w.WriteHeader(override) + return + } + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{"security_groups": list}) + }) + + mux.HandleFunc("/v2.0/security-groups/", func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodDelete { + w.WriteHeader(http.StatusMethodNotAllowed) + return + } + id := strings.TrimPrefix(r.URL.Path, "/v2.0/security-groups/") + srv.mu.Lock() + srv.deletedSGs = append(srv.deletedSGs, id) + status, ok := srv.deleteStatus[id] + if !ok { + status = http.StatusNoContent + } + srv.mu.Unlock() + w.WriteHeader(status) + }) + + srv.Server = httptest.NewServer(mux) + t.Cleanup(srv.Server.Close) + return srv +} + +func (srv *sgTestServer) authenticate(t *testing.T) *openstack.Session { + t.Helper() + cloudsYAML := fmt.Sprintf(` +clouds: + openstack: + auth_type: v3applicationcredential + auth: + auth_url: %s + application_credential_id: test-id + application_credential_secret: test-secret + interface: public + region_name: RegionOne +`, srv.URL) + session, err := openstack.Authenticate(context.Background(), cloudsYAML, "openstack", "") + if err != nil { + t.Fatalf("authenticate: %v", err) + } + if !session.IsAuthenticated() { + t.Fatal("expected authenticated session with network endpoint") + } + return session +} + +func sgDesc(cluster string) string { + return fmt.Sprintf("Security Group for Service LoadBalancer in cluster %s", cluster) +} + // Scenario: Pas d'endpoint "load-balancer" dans le catalogue → retour nil (LBs ignorés) func TestDeleteLoadBalancers_NoLoadBalancerEndpoint_Skips(t *testing.T) { ks := newKeystoneServer(t) @@ -797,3 +931,186 @@ func TestDeleteLoadBalancers_NoLoadBalancerEndpoint_Skips(t *testing.T) { t.Fatalf("expected nil when load-balancer endpoint is absent, got: %v", err) } } + +// ── Epic 4: Nettoyage des Security Groups ──────────────────────────────────── + +// ── US4.1: Identifier les Security Groups d'un cluster ─────────────────────── + +// Scenario: SG appartenant au cluster → inclus dans la suppression +// Scenario: SG d'un autre cluster → exclu +// Scenario: Description ne correspondant pas → exclu +func TestDeleteSecurityGroups_Filtering(t *testing.T) { + tests := []struct { + name string + description string + cluster string + shouldDelete bool + }{ + { + name: "matching cluster", + description: sgDesc("mycluster"), + cluster: "mycluster", + shouldDelete: true, + }, + { + name: "different cluster", + description: sgDesc("othercluster"), + cluster: "mycluster", + shouldDelete: false, + }, + { + name: "wrong prefix", + description: "Group for Service LoadBalancer in cluster mycluster", + cluster: "mycluster", + shouldDelete: false, + }, + { + name: "unrelated description", + description: "Some other security group", + cluster: "mycluster", + shouldDelete: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + srv := newSGTestServer(t) + sg := sgRecord{ID: "sg-001", Description: tt.description} + srv.sgLists = [][]sgRecord{{sg}, {}} + + session := srv.authenticate(t) + if err := session.DeleteSecurityGroups(context.Background(), logr.Discard(), tt.cluster); err != nil { + t.Fatalf("unexpected error: %v", err) + } + + srv.mu.Lock() + deleted := len(srv.deletedSGs) > 0 + srv.mu.Unlock() + + if deleted != tt.shouldDelete { + t.Errorf("shouldDelete=%v but SG was deleted=%v", tt.shouldDelete, deleted) + } + }) + } +} + +// ── US4.2: Supprimer les Security Groups ────────────────────────────────────── + +// Scenario: Suppression réussie +func TestDeleteSecurityGroups_SuccessfulDeletion(t *testing.T) { + srv := newSGTestServer(t) + sgs := []sgRecord{ + {ID: "sg-001", Description: sgDesc("mycluster")}, + {ID: "sg-002", Description: sgDesc("mycluster")}, + } + srv.sgLists = [][]sgRecord{sgs, {}} + + session := srv.authenticate(t) + if err := session.DeleteSecurityGroups(context.Background(), logr.Discard(), "mycluster"); err != nil { + t.Fatalf("unexpected error: %v", err) + } + + srv.mu.Lock() + deletedIDs := srv.deletedSGs + srv.mu.Unlock() + + if len(deletedIDs) != 2 { + t.Errorf("expected 2 SGs deleted, got %d: %v", len(deletedIDs), deletedIDs) + } + idSet := make(map[string]bool) + for _, id := range deletedIDs { + idSet[id] = true + } + for _, sg := range sgs { + if !idSet[sg.ID] { + t.Errorf("expected SG %s to be deleted", sg.ID) + } + } +} + +// Scenario: SG encore utilisé (HTTP 409) → warning, continue, vérification déclenchée +func TestDeleteSecurityGroups_TransientError409_ContinuesAndVerifies(t *testing.T) { + srv := newSGTestServer(t) + sgs := []sgRecord{ + {ID: "sg-001", Description: sgDesc("mycluster")}, // returns HTTP 409 + {ID: "sg-002", Description: sgDesc("mycluster")}, // returns HTTP 204 + } + srv.sgLists = [][]sgRecord{sgs, {}} + srv.deleteStatus["sg-001"] = http.StatusConflict + + session := srv.authenticate(t) + err := session.DeleteSecurityGroups(context.Background(), logr.Discard(), "mycluster") + + if err != nil { + t.Fatalf("expected no error for transient HTTP 409, got: %v", err) + } + + srv.mu.Lock() + attempted := len(srv.deletedSGs) + getCount := srv.sgGetCount + srv.mu.Unlock() + + if attempted != 2 { + t.Errorf("expected both SGs to be attempted, got %d DELETE calls", attempted) + } + // Verification GET must have been triggered (deleted=true even on transient error) + if getCount < 2 { + t.Errorf("expected at least 2 GET calls (list + verification), got %d", getCount) + } +} + +// Scenario: Erreur HTTP 500 → exception propagée +func TestDeleteSecurityGroups_HTTP500_PropagatesError(t *testing.T) { + srv := newSGTestServer(t) + srv.sgLists = [][]sgRecord{ + {{ID: "sg-server-error", Description: sgDesc("mycluster")}}, + } + srv.deleteStatus["sg-server-error"] = http.StatusInternalServerError + + session := srv.authenticate(t) + err := session.DeleteSecurityGroups(context.Background(), logr.Discard(), "mycluster") + + if err == nil { + t.Fatal("expected error for HTTP 500, got nil") + } +} + +// Scenario: SGs toujours présents après suppression → erreur retournée +func TestDeleteSecurityGroups_StillPresentAfterDeletion_ReturnsError(t *testing.T) { + srv := newSGTestServer(t) + sg := sgRecord{ID: "sg-persistent", Description: sgDesc("mycluster")} + srv.sgLists = [][]sgRecord{{sg}, {sg}} + + session := srv.authenticate(t) + err := session.DeleteSecurityGroups(context.Background(), logr.Discard(), "mycluster") + + if err == nil { + t.Fatal("expected error when SG persists after deletion") + } + if !strings.Contains(err.Error(), "mycluster") { + t.Errorf("expected error to mention cluster name, got: %v", err) + } +} + +// Scenario: Aucun SG correspondant → pas de suppression, pas de vérification +func TestDeleteSecurityGroups_NothingToDelete_NoVerification(t *testing.T) { + srv := newSGTestServer(t) + srv.sgLists = [][]sgRecord{{}} + + session := srv.authenticate(t) + if err := session.DeleteSecurityGroups(context.Background(), logr.Discard(), "mycluster"); err != nil { + t.Fatalf("unexpected error with no SGs: %v", err) + } + + srv.mu.Lock() + deletedCount := len(srv.deletedSGs) + getCount := srv.sgGetCount + srv.mu.Unlock() + + if deletedCount != 0 { + t.Errorf("expected no DELETE calls, got %d: %v", deletedCount, srv.deletedSGs) + } + if getCount != 1 { + t.Errorf("expected exactly 1 GET call (no verification), got %d", getCount) + } +} From 80a77f22e6a9a4edcf4920bcdaeccff8156ed983 Mon Sep 17 00:00:00 2001 From: Mathieu Grzybek Date: Wed, 8 Jul 2026 08:22:53 +0200 Subject: [PATCH 06/27] =?UTF-8?q?Epic=204=20=E2=80=94=20Security=20Groups?= =?UTF-8?q?=20(US4.1=20/=20US4.2):=20-=20resources=5Ftest.go:=206=20tests?= =?UTF-8?q?=20=E2=80=94=20description=20filter=20(prefix=20"Security=20Gro?= =?UTF-8?q?up=20=20=20for"=20+=20suffix=20"Service=20LoadBalancer=20in=20c?= =?UTF-8?q?luster=20"),=20transient=20HTTP=20=20=20409=20absorbed=20?= =?UTF-8?q?with=20verification=20triggered,=20HTTP=20500=20propagated,=20?= =?UTF-8?q?=20=20still-present=20=E2=86=92=20error,=20nothing-to-delete=20?= =?UTF-8?q?=E2=86=92=20no=20verification=20-=20resources.go=20=E2=80=94=20?= =?UTF-8?q?DeleteSecurityGroups:=20moved=20deleted=3Dtrue=20before=20DELET?= =?UTF-8?q?E=20=20=20call;=20added=20explicit=20warning=20log=20for=20tran?= =?UTF-8?q?sient=20errors?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Epic 5 — Cinder Volumes (US5.1): - resources_test.go: 6 tests — metadata filter (cinder.csi.openstack.org/cluster), keep annotation (janitor.capi.azimuth-cloud.com/keep=true) skips deletion, transient HTTP 409 absorbed with verification triggered, HTTP 500 propagated, still-present → error, nothing-to-delete → no verification - resources.go — DeleteVolumes: aligned on FIP/LB/SG pattern (deleted=true before DELETE, explicit warning log for transient errors) Signed-off-by: Mathieu Grzybek --- internal/openstack/resources.go | 10 +- internal/openstack/resources_test.go | 313 +++++++++++++++++++++++++++ 2 files changed, 320 insertions(+), 3 deletions(-) diff --git a/internal/openstack/resources.go b/internal/openstack/resources.go index 93278f4..002a3d1 100644 --- a/internal/openstack/resources.go +++ b/internal/openstack/resources.go @@ -216,11 +216,15 @@ func (s *Session) DeleteVolumes(ctx context.Context, logger logr.Logger, cluster if vol.Metadata[KeepProperty] == "true" { continue } + deleted = true logger.Info("deleting volume", "id", vol.ID) - if err := s.doDelete(ctx, cinderURL+"/volumes/"+vol.ID); err != nil && !isTransient(err) { - return err + if err := s.doDelete(ctx, cinderURL+"/volumes/"+vol.ID); err != nil { + if isTransient(err) { + logger.Info("transient error deleting volume, will verify", "id", vol.ID, "error", err) + } else { + return err + } } - deleted = true } if deleted { remaining, err := s.listVolumeItems(ctx, cinderURL+"/volumes/detail", "volumes") diff --git a/internal/openstack/resources_test.go b/internal/openstack/resources_test.go index abc89fc..bb3a532 100644 --- a/internal/openstack/resources_test.go +++ b/internal/openstack/resources_test.go @@ -1114,3 +1114,316 @@ func TestDeleteSecurityGroups_NothingToDelete_NoVerification(t *testing.T) { t.Errorf("expected exactly 1 GET call (no verification), got %d", getCount) } } + +// ── Cinder Volume mock server ───────────────────────────────────────────────── + +type cinderVolumeRecord struct { + ID string `json:"id"` + Metadata map[string]string `json:"metadata"` +} + +// cinderTestServer is a mock OpenStack server that handles Keystone auth + +// a Cinder-like volumes API. The catalog advertises the server's own URL as +// the "volumev3" endpoint (first type checked by cinderEndpoint). +type cinderTestServer struct { + *httptest.Server + mu sync.Mutex + volumeLists [][]cinderVolumeRecord // sequence of list responses; last entry is reused + volumeGetCount int + deleteStatus map[string]int + deletedVolumes []string +} + +func newCinderTestServer(t *testing.T) *cinderTestServer { + t.Helper() + srv := &cinderTestServer{ + deleteStatus: make(map[string]int), + } + mux := http.NewServeMux() + + mux.HandleFunc("/v3/auth/tokens", func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + w.WriteHeader(http.StatusMethodNotAllowed) + return + } + w.Header().Set("X-Subject-Token", "test-cinder-token") + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusCreated) + _ = json.NewEncoder(w).Encode(map[string]any{ + "token": map[string]any{"user": map[string]any{"id": "test-user"}}, + }) + }) + + // Self-referential catalog: "volumev3" endpoint = this server. + mux.HandleFunc("/v3/auth/catalog", func(w http.ResponseWriter, r *http.Request) { + selfURL := "http://" + r.Host + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{ + "catalog": []any{ + map[string]any{ + "type": "volumev3", + "endpoints": []any{ + map[string]any{ + "interface": "public", + "region_id": "RegionOne", + "url": selfURL, + }, + }, + }, + }, + }) + }) + + // Cinder: list volumes (exact path — takes priority over /volumes/ subtree) + mux.HandleFunc("/volumes/detail", func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + w.WriteHeader(http.StatusMethodNotAllowed) + return + } + srv.mu.Lock() + srv.volumeGetCount++ + idx := srv.volumeGetCount - 1 + var list []cinderVolumeRecord + if len(srv.volumeLists) > 0 { + if idx < len(srv.volumeLists) { + list = srv.volumeLists[idx] + } else { + list = srv.volumeLists[len(srv.volumeLists)-1] + } + } + srv.mu.Unlock() + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{"volumes": list}) + }) + + // Cinder: delete volume by ID (subtree pattern handles /volumes/{id}) + mux.HandleFunc("/volumes/", func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodDelete { + w.WriteHeader(http.StatusMethodNotAllowed) + return + } + id := strings.TrimPrefix(r.URL.Path, "/volumes/") + srv.mu.Lock() + srv.deletedVolumes = append(srv.deletedVolumes, id) + status, ok := srv.deleteStatus[id] + if !ok { + status = http.StatusNoContent + } + srv.mu.Unlock() + w.WriteHeader(status) + }) + + srv.Server = httptest.NewServer(mux) + t.Cleanup(srv.Server.Close) + return srv +} + +func (srv *cinderTestServer) authenticate(t *testing.T) *openstack.Session { + t.Helper() + cloudsYAML := fmt.Sprintf(` +clouds: + openstack: + auth_type: v3applicationcredential + auth: + auth_url: %s + application_credential_id: test-id + application_credential_secret: test-secret + interface: public + region_name: RegionOne +`, srv.URL) + session, err := openstack.Authenticate(context.Background(), cloudsYAML, "openstack", "") + if err != nil { + t.Fatalf("authenticate: %v", err) + } + if !session.IsAuthenticated() { + t.Fatal("expected authenticated session with volumev3 endpoint") + } + return session +} + +// ── Epic 5: Gestion des Volumes Cinder ─────────────────────────────────────── + +// ── US5.1: Identifier les volumes d'un cluster ─────────────────────────────── + +// Scenario: Volume du bon cluster sans keep → supprimé +// Scenario: Volume avec keep=true → conservé +// Scenario: Volume d'un autre cluster → exclu +// Scenario: Volume sans métadonnée CSI → exclu +func TestDeleteVolumes_Filtering(t *testing.T) { + tests := []struct { + name string + metadata map[string]string + cluster string + shouldDelete bool + }{ + { + name: "matching cluster, no keep", + metadata: map[string]string{"cinder.csi.openstack.org/cluster": "mycluster"}, + cluster: "mycluster", + shouldDelete: true, + }, + { + name: "matching cluster, keep=true", + metadata: map[string]string{ + "cinder.csi.openstack.org/cluster": "mycluster", + "janitor.capi.azimuth-cloud.com/keep": "true", + }, + cluster: "mycluster", + shouldDelete: false, + }, + { + name: "different cluster", + metadata: map[string]string{"cinder.csi.openstack.org/cluster": "othercluster"}, + cluster: "mycluster", + shouldDelete: false, + }, + { + name: "no CSI metadata", + metadata: map[string]string{}, + cluster: "mycluster", + shouldDelete: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + srv := newCinderTestServer(t) + vol := cinderVolumeRecord{ID: "vol-001", Metadata: tt.metadata} + srv.volumeLists = [][]cinderVolumeRecord{{vol}, {}} + + session := srv.authenticate(t) + if err := session.DeleteVolumes(context.Background(), logr.Discard(), tt.cluster); err != nil { + t.Fatalf("unexpected error: %v", err) + } + + srv.mu.Lock() + deleted := len(srv.deletedVolumes) > 0 + srv.mu.Unlock() + + if deleted != tt.shouldDelete { + t.Errorf("shouldDelete=%v but volume was deleted=%v", tt.shouldDelete, deleted) + } + }) + } +} + +// Scenario: Suppression réussie de plusieurs volumes +func TestDeleteVolumes_SuccessfulDeletion(t *testing.T) { + srv := newCinderTestServer(t) + vols := []cinderVolumeRecord{ + {ID: "vol-001", Metadata: map[string]string{"cinder.csi.openstack.org/cluster": "mycluster"}}, + {ID: "vol-002", Metadata: map[string]string{"cinder.csi.openstack.org/cluster": "mycluster"}}, + } + srv.volumeLists = [][]cinderVolumeRecord{vols, {}} + + session := srv.authenticate(t) + if err := session.DeleteVolumes(context.Background(), logr.Discard(), "mycluster"); err != nil { + t.Fatalf("unexpected error: %v", err) + } + + srv.mu.Lock() + deletedIDs := srv.deletedVolumes + srv.mu.Unlock() + + if len(deletedIDs) != 2 { + t.Errorf("expected 2 volumes deleted, got %d: %v", len(deletedIDs), deletedIDs) + } + idSet := make(map[string]bool) + for _, id := range deletedIDs { + idSet[id] = true + } + for _, vol := range vols { + if !idSet[vol.ID] { + t.Errorf("expected volume %s to be deleted", vol.ID) + } + } +} + +// Scenario: Erreur HTTP 409 (transiente) → warning, continue, vérification déclenchée +func TestDeleteVolumes_TransientError409_ContinuesAndVerifies(t *testing.T) { + srv := newCinderTestServer(t) + vols := []cinderVolumeRecord{ + {ID: "vol-001", Metadata: map[string]string{"cinder.csi.openstack.org/cluster": "mycluster"}}, + {ID: "vol-002", Metadata: map[string]string{"cinder.csi.openstack.org/cluster": "mycluster"}}, + } + srv.volumeLists = [][]cinderVolumeRecord{vols, {}} + srv.deleteStatus["vol-001"] = http.StatusConflict + + session := srv.authenticate(t) + err := session.DeleteVolumes(context.Background(), logr.Discard(), "mycluster") + + if err != nil { + t.Fatalf("expected no error for transient HTTP 409, got: %v", err) + } + + srv.mu.Lock() + attempted := len(srv.deletedVolumes) + getCount := srv.volumeGetCount + srv.mu.Unlock() + + if attempted != 2 { + t.Errorf("expected both volumes to be attempted, got %d DELETE calls", attempted) + } + if getCount < 2 { + t.Errorf("expected at least 2 GET calls (list + verification), got %d", getCount) + } +} + +// Scenario: Erreur HTTP 500 → exception propagée +func TestDeleteVolumes_HTTP500_PropagatesError(t *testing.T) { + srv := newCinderTestServer(t) + srv.volumeLists = [][]cinderVolumeRecord{ + {{ID: "vol-server-error", Metadata: map[string]string{"cinder.csi.openstack.org/cluster": "mycluster"}}}, + } + srv.deleteStatus["vol-server-error"] = http.StatusInternalServerError + + session := srv.authenticate(t) + err := session.DeleteVolumes(context.Background(), logr.Discard(), "mycluster") + + if err == nil { + t.Fatal("expected error for HTTP 500, got nil") + } +} + +// Scenario: Volume toujours présent après suppression → erreur retournée +func TestDeleteVolumes_StillPresentAfterDeletion_ReturnsError(t *testing.T) { + srv := newCinderTestServer(t) + vol := cinderVolumeRecord{ + ID: "vol-persistent", + Metadata: map[string]string{"cinder.csi.openstack.org/cluster": "mycluster"}, + } + srv.volumeLists = [][]cinderVolumeRecord{{vol}, {vol}} + + session := srv.authenticate(t) + err := session.DeleteVolumes(context.Background(), logr.Discard(), "mycluster") + + if err == nil { + t.Fatal("expected error when volume persists after deletion") + } + if !strings.Contains(err.Error(), "mycluster") { + t.Errorf("expected error to mention cluster name, got: %v", err) + } +} + +// Scenario: Aucun volume correspondant → pas de suppression, pas de vérification +func TestDeleteVolumes_NothingToDelete_NoVerification(t *testing.T) { + srv := newCinderTestServer(t) + srv.volumeLists = [][]cinderVolumeRecord{{}} + + session := srv.authenticate(t) + if err := session.DeleteVolumes(context.Background(), logr.Discard(), "mycluster"); err != nil { + t.Fatalf("unexpected error with no volumes: %v", err) + } + + srv.mu.Lock() + deletedCount := len(srv.deletedVolumes) + getCount := srv.volumeGetCount + srv.mu.Unlock() + + if deletedCount != 0 { + t.Errorf("expected no DELETE calls, got %d: %v", deletedCount, srv.deletedVolumes) + } + if getCount != 1 { + t.Errorf("expected exactly 1 GET call (no verification), got %d", getCount) + } +} From 0035f759be29320818ead74387eb687829995f53 Mon Sep 17 00:00:00 2001 From: Mathieu Grzybek Date: Wed, 8 Jul 2026 08:25:38 +0200 Subject: [PATCH 07/27] =?UTF-8?q?feat(openstack):=20Epic=206=20=E2=80=94?= =?UTF-8?q?=20Cinder=20Snapshot=20cleanup=20(TDD)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - resources_test.go: 6 tests covering US6.1 — CSI metadata filter (cinder.csi.openstack.org/cluster), transient HTTP 409 absorbed with verification triggered, HTTP 500 propagated, still-present → error, nothing-to-delete → no verification - resources.go — DeleteSnapshots: aligned on FIP/LB/SG/Volume pattern (deleted=true before DELETE, explicit warning log for transient errors) Signed-off-by: Mathieu Grzybek --- internal/openstack/resources.go | 10 +- internal/openstack/resources_test.go | 296 +++++++++++++++++++++++++++ 2 files changed, 303 insertions(+), 3 deletions(-) diff --git a/internal/openstack/resources.go b/internal/openstack/resources.go index 002a3d1..78c98d9 100644 --- a/internal/openstack/resources.go +++ b/internal/openstack/resources.go @@ -175,11 +175,15 @@ func (s *Session) DeleteSnapshots(ctx context.Context, logger logr.Logger, clust deleted := false for _, snap := range list { if snap.Metadata["cinder.csi.openstack.org/cluster"] == cluster { + deleted = true logger.Info("deleting snapshot", "id", snap.ID) - if err := s.doDelete(ctx, cinderURL+"/snapshots/"+snap.ID); err != nil && !isTransient(err) { - return err + if err := s.doDelete(ctx, cinderURL+"/snapshots/"+snap.ID); err != nil { + if isTransient(err) { + logger.Info("transient error deleting snapshot, will verify", "id", snap.ID, "error", err) + } else { + return err + } } - deleted = true } } if deleted { diff --git a/internal/openstack/resources_test.go b/internal/openstack/resources_test.go index bb3a532..7fa08e2 100644 --- a/internal/openstack/resources_test.go +++ b/internal/openstack/resources_test.go @@ -1427,3 +1427,299 @@ func TestDeleteVolumes_NothingToDelete_NoVerification(t *testing.T) { t.Errorf("expected exactly 1 GET call (no verification), got %d", getCount) } } + +// ── Cinder Snapshot mock server ─────────────────────────────────────────────── + +// snapshotTestServer is a mock OpenStack server handling Keystone auth + +// a Cinder-like snapshots API. Catalog advertises the server's own URL as +// the "volumev3" endpoint (same as volumes — same Cinder service). +type snapshotTestServer struct { + *httptest.Server + mu sync.Mutex + snapshotLists [][]cinderVolumeRecord // reuse volumeItem shape (id + metadata) + snapshotGetCount int + deleteStatus map[string]int + deletedSnapshots []string +} + +func newSnapshotTestServer(t *testing.T) *snapshotTestServer { + t.Helper() + srv := &snapshotTestServer{ + deleteStatus: make(map[string]int), + } + mux := http.NewServeMux() + + mux.HandleFunc("/v3/auth/tokens", func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + w.WriteHeader(http.StatusMethodNotAllowed) + return + } + w.Header().Set("X-Subject-Token", "test-snapshot-token") + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusCreated) + _ = json.NewEncoder(w).Encode(map[string]any{ + "token": map[string]any{"user": map[string]any{"id": "test-user"}}, + }) + }) + + mux.HandleFunc("/v3/auth/catalog", func(w http.ResponseWriter, r *http.Request) { + selfURL := "http://" + r.Host + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{ + "catalog": []any{ + map[string]any{ + "type": "volumev3", + "endpoints": []any{ + map[string]any{ + "interface": "public", + "region_id": "RegionOne", + "url": selfURL, + }, + }, + }, + }, + }) + }) + + // Cinder: list snapshots (exact path — takes priority over /snapshots/ subtree) + mux.HandleFunc("/snapshots/detail", func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + w.WriteHeader(http.StatusMethodNotAllowed) + return + } + srv.mu.Lock() + srv.snapshotGetCount++ + idx := srv.snapshotGetCount - 1 + var list []cinderVolumeRecord + if len(srv.snapshotLists) > 0 { + if idx < len(srv.snapshotLists) { + list = srv.snapshotLists[idx] + } else { + list = srv.snapshotLists[len(srv.snapshotLists)-1] + } + } + srv.mu.Unlock() + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{"snapshots": list}) + }) + + // Cinder: delete snapshot by ID (subtree pattern handles /snapshots/{id}) + mux.HandleFunc("/snapshots/", func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodDelete { + w.WriteHeader(http.StatusMethodNotAllowed) + return + } + id := strings.TrimPrefix(r.URL.Path, "/snapshots/") + srv.mu.Lock() + srv.deletedSnapshots = append(srv.deletedSnapshots, id) + status, ok := srv.deleteStatus[id] + if !ok { + status = http.StatusNoContent + } + srv.mu.Unlock() + w.WriteHeader(status) + }) + + srv.Server = httptest.NewServer(mux) + t.Cleanup(srv.Server.Close) + return srv +} + +func (srv *snapshotTestServer) authenticate(t *testing.T) *openstack.Session { + t.Helper() + cloudsYAML := fmt.Sprintf(` +clouds: + openstack: + auth_type: v3applicationcredential + auth: + auth_url: %s + application_credential_id: test-id + application_credential_secret: test-secret + interface: public + region_name: RegionOne +`, srv.URL) + session, err := openstack.Authenticate(context.Background(), cloudsYAML, "openstack", "") + if err != nil { + t.Fatalf("authenticate: %v", err) + } + if !session.IsAuthenticated() { + t.Fatal("expected authenticated session with volumev3 endpoint") + } + return session +} + +// ── Epic 6: Gestion des Snapshots Cinder ───────────────────────────────────── + +// ── US6.1: Identifier et supprimer les snapshots d'un cluster ──────────────── + +// Scenario: Snapshot du bon cluster → supprimé +// Scenario: Snapshot d'un autre cluster → exclu +func TestDeleteSnapshots_Filtering(t *testing.T) { + tests := []struct { + name string + metadata map[string]string + cluster string + shouldDelete bool + }{ + { + name: "matching cluster", + metadata: map[string]string{"cinder.csi.openstack.org/cluster": "mycluster"}, + cluster: "mycluster", + shouldDelete: true, + }, + { + name: "different cluster", + metadata: map[string]string{"cinder.csi.openstack.org/cluster": "othercluster"}, + cluster: "mycluster", + shouldDelete: false, + }, + { + name: "no CSI metadata", + metadata: map[string]string{}, + cluster: "mycluster", + shouldDelete: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + srv := newSnapshotTestServer(t) + snap := cinderVolumeRecord{ID: "snap-001", Metadata: tt.metadata} + srv.snapshotLists = [][]cinderVolumeRecord{{snap}, {}} + + session := srv.authenticate(t) + if err := session.DeleteSnapshots(context.Background(), logr.Discard(), tt.cluster); err != nil { + t.Fatalf("unexpected error: %v", err) + } + + srv.mu.Lock() + deleted := len(srv.deletedSnapshots) > 0 + srv.mu.Unlock() + + if deleted != tt.shouldDelete { + t.Errorf("shouldDelete=%v but snapshot was deleted=%v", tt.shouldDelete, deleted) + } + }) + } +} + +// Scenario: Suppression réussie de plusieurs snapshots +func TestDeleteSnapshots_SuccessfulDeletion(t *testing.T) { + srv := newSnapshotTestServer(t) + snaps := []cinderVolumeRecord{ + {ID: "snap-001", Metadata: map[string]string{"cinder.csi.openstack.org/cluster": "mycluster"}}, + {ID: "snap-002", Metadata: map[string]string{"cinder.csi.openstack.org/cluster": "mycluster"}}, + } + srv.snapshotLists = [][]cinderVolumeRecord{snaps, {}} + + session := srv.authenticate(t) + if err := session.DeleteSnapshots(context.Background(), logr.Discard(), "mycluster"); err != nil { + t.Fatalf("unexpected error: %v", err) + } + + srv.mu.Lock() + deletedIDs := srv.deletedSnapshots + srv.mu.Unlock() + + if len(deletedIDs) != 2 { + t.Errorf("expected 2 snapshots deleted, got %d: %v", len(deletedIDs), deletedIDs) + } + idSet := make(map[string]bool) + for _, id := range deletedIDs { + idSet[id] = true + } + for _, snap := range snaps { + if !idSet[snap.ID] { + t.Errorf("expected snapshot %s to be deleted", snap.ID) + } + } +} + +// Scenario: Erreur HTTP 409 (transiente) → warning, continue, vérification déclenchée +func TestDeleteSnapshots_TransientError409_ContinuesAndVerifies(t *testing.T) { + srv := newSnapshotTestServer(t) + snaps := []cinderVolumeRecord{ + {ID: "snap-001", Metadata: map[string]string{"cinder.csi.openstack.org/cluster": "mycluster"}}, + {ID: "snap-002", Metadata: map[string]string{"cinder.csi.openstack.org/cluster": "mycluster"}}, + } + srv.snapshotLists = [][]cinderVolumeRecord{snaps, {}} + srv.deleteStatus["snap-001"] = http.StatusConflict + + session := srv.authenticate(t) + err := session.DeleteSnapshots(context.Background(), logr.Discard(), "mycluster") + + if err != nil { + t.Fatalf("expected no error for transient HTTP 409, got: %v", err) + } + + srv.mu.Lock() + attempted := len(srv.deletedSnapshots) + getCount := srv.snapshotGetCount + srv.mu.Unlock() + + if attempted != 2 { + t.Errorf("expected both snapshots to be attempted, got %d DELETE calls", attempted) + } + if getCount < 2 { + t.Errorf("expected at least 2 GET calls (list + verification), got %d", getCount) + } +} + +// Scenario: Erreur HTTP 500 → exception propagée +func TestDeleteSnapshots_HTTP500_PropagatesError(t *testing.T) { + srv := newSnapshotTestServer(t) + srv.snapshotLists = [][]cinderVolumeRecord{ + {{ID: "snap-server-error", Metadata: map[string]string{"cinder.csi.openstack.org/cluster": "mycluster"}}}, + } + srv.deleteStatus["snap-server-error"] = http.StatusInternalServerError + + session := srv.authenticate(t) + err := session.DeleteSnapshots(context.Background(), logr.Discard(), "mycluster") + + if err == nil { + t.Fatal("expected error for HTTP 500, got nil") + } +} + +// Scenario: Snapshot toujours présent après suppression → erreur retournée +func TestDeleteSnapshots_StillPresentAfterDeletion_ReturnsError(t *testing.T) { + srv := newSnapshotTestServer(t) + snap := cinderVolumeRecord{ + ID: "snap-persistent", + Metadata: map[string]string{"cinder.csi.openstack.org/cluster": "mycluster"}, + } + srv.snapshotLists = [][]cinderVolumeRecord{{snap}, {snap}} + + session := srv.authenticate(t) + err := session.DeleteSnapshots(context.Background(), logr.Discard(), "mycluster") + + if err == nil { + t.Fatal("expected error when snapshot persists after deletion") + } + if !strings.Contains(err.Error(), "mycluster") { + t.Errorf("expected error to mention cluster name, got: %v", err) + } +} + +// Scenario: Aucun snapshot correspondant → pas de suppression, pas de vérification +func TestDeleteSnapshots_NothingToDelete_NoVerification(t *testing.T) { + srv := newSnapshotTestServer(t) + srv.snapshotLists = [][]cinderVolumeRecord{{}} + + session := srv.authenticate(t) + if err := session.DeleteSnapshots(context.Background(), logr.Discard(), "mycluster"); err != nil { + t.Fatalf("unexpected error with no snapshots: %v", err) + } + + srv.mu.Lock() + deletedCount := len(srv.deletedSnapshots) + getCount := srv.snapshotGetCount + srv.mu.Unlock() + + if deletedCount != 0 { + t.Errorf("expected no DELETE calls, got %d: %v", deletedCount, srv.deletedSnapshots) + } + if getCount != 1 { + t.Errorf("expected exactly 1 GET call (no verification), got %d", getCount) + } +} From 692654a01752cfeec91b81f883ff3f854f1b1bb9 Mon Sep 17 00:00:00 2001 From: Mathieu Grzybek Date: Wed, 8 Jul 2026 08:28:17 +0200 Subject: [PATCH 08/27] =?UTF-8?q?feat(openstack):=20Epic=207=20=E2=80=94?= =?UTF-8?q?=20Application=20Credential=20cleanup=20(TDD)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - resources_test.go: 5 tests covering US7.1 — successful deletion (204), already-deleted (404 → nil), forbidden (403 → graceful skip), HTTP 500 propagated, no identity endpoint → CatalogError - resources.go: no changes needed (implementation already correct) Signed-off-by: Mathieu Grzybek --- internal/openstack/resources_test.go | 181 +++++++++++++++++++++++++++ 1 file changed, 181 insertions(+) diff --git a/internal/openstack/resources_test.go b/internal/openstack/resources_test.go index 7fa08e2..bb599a6 100644 --- a/internal/openstack/resources_test.go +++ b/internal/openstack/resources_test.go @@ -1701,6 +1701,187 @@ func TestDeleteSnapshots_StillPresentAfterDeletion_ReturnsError(t *testing.T) { } } +// ── Identity (AppCredential) mock server ────────────────────────────────────── + +// identityTestServer is a mock OpenStack server handling Keystone auth + +// the Identity API for application credentials. The catalog advertises the +// server's own URL as the "identity" endpoint. +type identityTestServer struct { + *httptest.Server + mu sync.Mutex + deleteStatus int // HTTP status for the DELETE request; 0 means 204 + deletedAppcredID string +} + +func newIdentityTestServer(t *testing.T) *identityTestServer { + t.Helper() + srv := &identityTestServer{} + mux := http.NewServeMux() + + mux.HandleFunc("/v3/auth/tokens", func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + w.WriteHeader(http.StatusMethodNotAllowed) + return + } + w.Header().Set("X-Subject-Token", "test-identity-token") + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusCreated) + _ = json.NewEncoder(w).Encode(map[string]any{ + "token": map[string]any{"user": map[string]any{"id": "test-user"}}, + }) + }) + + // Self-referential catalog: "identity" endpoint = this server. + mux.HandleFunc("/v3/auth/catalog", func(w http.ResponseWriter, r *http.Request) { + selfURL := "http://" + r.Host + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{ + "catalog": []any{ + map[string]any{ + "type": "identity", + "endpoints": []any{ + map[string]any{ + "interface": "public", + "region_id": "RegionOne", + "url": selfURL, + }, + }, + }, + }, + }) + }) + + // Identity: DELETE /v3/users/{userID}/application_credentials/{appcredID} + mux.HandleFunc("/v3/users/", func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodDelete { + w.WriteHeader(http.StatusMethodNotAllowed) + return + } + parts := strings.Split(r.URL.Path, "/application_credentials/") + srv.mu.Lock() + if len(parts) == 2 { + srv.deletedAppcredID = parts[1] + } + status := srv.deleteStatus + if status == 0 { + status = http.StatusNoContent + } + srv.mu.Unlock() + w.WriteHeader(status) + }) + + srv.Server = httptest.NewServer(mux) + t.Cleanup(srv.Server.Close) + return srv +} + +// authenticate returns both the session and the cloudsYAML used to build it, +// so the same YAML can be passed to DeleteAppCredential. +func (srv *identityTestServer) authenticate(t *testing.T) (*openstack.Session, string) { + t.Helper() + cloudsYAML := fmt.Sprintf(` +clouds: + openstack: + auth_type: v3applicationcredential + auth: + auth_url: %s + application_credential_id: test-appcred-id + application_credential_secret: test-secret + interface: public + region_name: RegionOne +`, srv.URL) + session, err := openstack.Authenticate(context.Background(), cloudsYAML, "openstack", "") + if err != nil { + t.Fatalf("authenticate: %v", err) + } + if !session.IsAuthenticated() { + t.Fatal("expected authenticated session with identity endpoint") + } + return session, cloudsYAML +} + +// ── Epic 7: Gestion des Application Credentials ─────────────────────────────── + +// ── US7.1: Supprimer l'Application Credential OpenStack ────────────────────── + +// Scenario: Suppression autorisée → HTTP 204, retour nil +func TestDeleteAppCredential_SuccessfulDeletion(t *testing.T) { + srv := newIdentityTestServer(t) + session, cloudsYAML := srv.authenticate(t) + + if err := session.DeleteAppCredential(context.Background(), logr.Discard(), cloudsYAML, "openstack"); err != nil { + t.Fatalf("unexpected error: %v", err) + } + + srv.mu.Lock() + deleted := srv.deletedAppcredID + srv.mu.Unlock() + + if deleted != "test-appcred-id" { + t.Errorf("expected appcred-id %q to be deleted, got %q", "test-appcred-id", deleted) + } +} + +// Scenario: Application Credential déjà supprimé → HTTP 404, retour nil +func TestDeleteAppCredential_AlreadyDeleted_404(t *testing.T) { + srv := newIdentityTestServer(t) + srv.deleteStatus = http.StatusNotFound + session, cloudsYAML := srv.authenticate(t) + + if err := session.DeleteAppCredential(context.Background(), logr.Discard(), cloudsYAML, "openstack"); err != nil { + t.Fatalf("expected nil for HTTP 404 (already deleted), got: %v", err) + } +} + +// Scenario: Application Credential non supprimable (403) → warning, retour nil +func TestDeleteAppCredential_Forbidden_403(t *testing.T) { + srv := newIdentityTestServer(t) + srv.deleteStatus = http.StatusForbidden + session, cloudsYAML := srv.authenticate(t) + + if err := session.DeleteAppCredential(context.Background(), logr.Discard(), cloudsYAML, "openstack"); err != nil { + t.Fatalf("expected nil for HTTP 403 (restricted), got: %v", err) + } +} + +// Scenario: Erreur HTTP 500 → erreur propagée +func TestDeleteAppCredential_HTTP500_PropagatesError(t *testing.T) { + srv := newIdentityTestServer(t) + srv.deleteStatus = http.StatusInternalServerError + session, cloudsYAML := srv.authenticate(t) + + err := session.DeleteAppCredential(context.Background(), logr.Discard(), cloudsYAML, "openstack") + if err == nil { + t.Fatal("expected error for HTTP 500, got nil") + } +} + +// Scenario: Pas d'endpoint "identity" dans le catalogue → CatalogError +func TestDeleteAppCredential_NoIdentityEndpoint_ReturnsCatalogError(t *testing.T) { + ks := newKeystoneServer(t) + ks.catalog = catalogWith( + serviceEntry("network", + endpoint("public", "RegionOne", "http://network.example.com"), + ), + ) + + clouds := buildCloudsYAML(ks.URL, "v3applicationcredential") + session, err := openstack.Authenticate(context.Background(), clouds, "openstack", "") + if err != nil { + t.Fatalf("unexpected authenticate error: %v", err) + } + + err = session.DeleteAppCredential(context.Background(), logr.Discard(), clouds, "openstack") + + if err == nil { + t.Fatal("expected CatalogError when identity endpoint is absent") + } + var target *openstack.CatalogError + if !errorAs(err, &target) { + t.Errorf("expected *CatalogError, got %T: %v", err, err) + } +} + // Scenario: Aucun snapshot correspondant → pas de suppression, pas de vérification func TestDeleteSnapshots_NothingToDelete_NoVerification(t *testing.T) { srv := newSnapshotTestServer(t) From 51b934c9e986e578e5fc2bda316e206038168371 Mon Sep 17 00:00:00 2001 From: Mathieu Grzybek Date: Wed, 8 Jul 2026 08:43:53 +0200 Subject: [PATCH 09/27] =?UTF-8?q?feat(controller):=20Epic=208=20=E2=80=94?= =?UTF-8?q?=20Kubernetes=20lifecycle=20(TDD)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - controller.go: add PurgeFunc/SleepFunc injectable fields; replace openstack.PurgeResources and time.Sleep calls with r.purge/r.sleep helpers (defaults preserved, no behavior change in production) - openstackcluster_controller_test.go: 8 tests covering US8.1 (finalizer add when absent, idempotent when present), US8.2 (cluster name from label vs metadata.name), US8.3 (finalizer removed after successful purge, cleanup skipped when finalizer absent), US8.4 (retry annotation set on purge error, NotFound ignored when cluster deleted during retry) Signed-off-by: Mathieu Grzybek --- .../controller/openstackcluster_controller.go | 26 +- .../openstackcluster_controller_test.go | 285 ++++++++++++++++++ 2 files changed, 307 insertions(+), 4 deletions(-) create mode 100644 internal/controller/openstackcluster_controller_test.go diff --git a/internal/controller/openstackcluster_controller.go b/internal/controller/openstackcluster_controller.go index 93995ce..6ef1503 100644 --- a/internal/controller/openstackcluster_controller.go +++ b/internal/controller/openstackcluster_controller.go @@ -57,6 +57,25 @@ type OpenStackClusterReconciler struct { Scheme *runtime.Scheme DefaultVolumesPolicy string RetryDefaultDelay int + // PurgeFunc is called to clean up OpenStack resources; defaults to openstack.PurgeResources. + PurgeFunc func(context.Context, openstack.PurgeOptions) error + // SleepFunc is called instead of time.Sleep; defaults to time.Sleep. + SleepFunc func(time.Duration) +} + +func (r *OpenStackClusterReconciler) purge(ctx context.Context, opts openstack.PurgeOptions) error { + if r.PurgeFunc != nil { + return r.PurgeFunc(ctx, opts) + } + return openstack.PurgeResources(ctx, opts) +} + +func (r *OpenStackClusterReconciler) sleep(d time.Duration) { + if r.SleepFunc != nil { + r.SleepFunc(d) + } else { + time.Sleep(d) + } } //+kubebuilder:rbac:groups=infrastructure.cluster.x-k8s.io,resources=openstackclusters,verbs=get;list;watch;patch;update @@ -118,7 +137,7 @@ func (r *OpenStackClusterReconciler) Reconcile(ctx context.Context, req ctrl.Req credentialPolicy := secret.Annotations[CredentialPolicyAnnotation] includeAppcred := credentialPolicy == PolicyDelete && len(cluster.Finalizers) == 1 - purgeErr := openstack.PurgeResources(ctx, openstack.PurgeOptions{ + purgeErr := r.purge(ctx, openstack.PurgeOptions{ CloudsYAML: cloudsYAML, CloudName: cloudName, CACert: cacert, @@ -129,8 +148,7 @@ func (r *OpenStackClusterReconciler) Reconcile(ctx context.Context, req ctrl.Req }) if purgeErr != nil { logger.Error(purgeErr, "purge failed, will retry") - delay := r.retryDelay() - time.Sleep(delay) + r.sleep(r.retryDelay()) if err := r.annotateRetry(ctx, &cluster); err != nil && !apierrors.IsNotFound(err) { return ctrl.Result{}, err } @@ -148,7 +166,7 @@ func (r *OpenStackClusterReconciler) Reconcile(ctx context.Context, req ctrl.Req // Other finalizers still present; trigger a retry when they are removed. other := otherFinalizer(cluster.Finalizers, Finalizer) logger.Info("waiting for other finalizer before deleting appcred", "otherFinalizer", other) - time.Sleep(5 * time.Second) + r.sleep(5 * time.Second) if err := r.annotateRetry(ctx, &cluster); err != nil && !apierrors.IsNotFound(err) { return ctrl.Result{}, err } diff --git a/internal/controller/openstackcluster_controller_test.go b/internal/controller/openstackcluster_controller_test.go new file mode 100644 index 0000000..14edff1 --- /dev/null +++ b/internal/controller/openstackcluster_controller_test.go @@ -0,0 +1,285 @@ +package controller_test + +import ( + "context" + "errors" + "fmt" + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + infrav1 "sigs.k8s.io/cluster-api-provider-openstack/api/v1beta1" + + "github.com/azimuth-cloud/cluster-api-janitor-openstack/internal/controller" + "github.com/azimuth-cloud/cluster-api-janitor-openstack/internal/openstack" +) + +var testScheme *runtime.Scheme + +func init() { + testScheme = runtime.NewScheme() + _ = clientgoscheme.AddToScheme(testScheme) + _ = infrav1.AddToScheme(testScheme) +} + +// ── Helpers ─────────────────────────────────────────────────────────────────── + +func reconcileRequest(name, namespace string) ctrl.Request { + return ctrl.Request{NamespacedName: types.NamespacedName{Name: name, Namespace: namespace}} +} + +func newReconciler(purgeFunc func(context.Context, openstack.PurgeOptions) error, objs ...client.Object) (*controller.OpenStackClusterReconciler, client.Client) { + c := fake.NewClientBuilder().WithScheme(testScheme).WithObjects(objs...).Build() + r := &controller.OpenStackClusterReconciler{ + Client: c, + Scheme: testScheme, + PurgeFunc: purgeFunc, + SleepFunc: func(time.Duration) {}, // no-op: avoid real sleeps in tests + } + return r, c +} + +func newCluster(name, namespace string, opts ...func(*infrav1.OpenStackCluster)) *infrav1.OpenStackCluster { + c := &infrav1.OpenStackCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Spec: infrav1.OpenStackClusterSpec{ + IdentityRef: infrav1.OpenStackIdentityReference{ + Name: "cloud-credentials", + CloudName: "openstack", + }, + }, + } + for _, opt := range opts { + opt(c) + } + return c +} + +func withFinalizer(c *infrav1.OpenStackCluster) { + controllerutil.AddFinalizer(c, controller.Finalizer) +} + +func withDeletionTimestamp(c *infrav1.OpenStackCluster) { + now := metav1.Now() + c.DeletionTimestamp = &now +} + +func withClusterLabel(value string) func(*infrav1.OpenStackCluster) { + return func(c *infrav1.OpenStackCluster) { + if c.Labels == nil { + c.Labels = make(map[string]string) + } + c.Labels[controller.ClusterNameLabel] = value + } +} + +func newSecret(name, namespace string) *corev1.Secret { + return &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: namespace}, + Data: map[string][]byte{"clouds.yaml": []byte("clouds: {}")}, + } +} + +func getClusterOrNil(t *testing.T, c client.Client, name, namespace string) *infrav1.OpenStackCluster { + t.Helper() + var cluster infrav1.OpenStackCluster + err := c.Get(context.Background(), types.NamespacedName{Name: name, Namespace: namespace}, &cluster) + if apierrors.IsNotFound(err) { + return nil + } + if err != nil { + t.Fatalf("getting cluster: %v", err) + } + return &cluster +} + +// ── US8.1: Ajouter un finalizer ─────────────────────────────────────────────── + +// Scenario: Cluster sans deletionTimestamp et sans finalizer → finalizer ajouté +func TestReconcile_AddsFinalizer_WhenNotPresent(t *testing.T) { + cluster := newCluster("mycluster", "default") + r, c := newReconciler(nil, cluster) + + if _, err := r.Reconcile(context.Background(), reconcileRequest("mycluster", "default")); err != nil { + t.Fatalf("unexpected error: %v", err) + } + + got := getClusterOrNil(t, c, "mycluster", "default") + if got == nil { + t.Fatal("cluster not found after reconcile") + } + if !controllerutil.ContainsFinalizer(got, controller.Finalizer) { + t.Errorf("expected finalizer %q to be added, got finalizers: %v", controller.Finalizer, got.Finalizers) + } +} + +// Scenario: Cluster avec finalizer déjà présent → aucun changement d'état +func TestReconcile_FinalizerAlreadyPresent_Idempotent(t *testing.T) { + cluster := newCluster("mycluster", "default", withFinalizer) + r, c := newReconciler(nil, cluster) + + if _, err := r.Reconcile(context.Background(), reconcileRequest("mycluster", "default")); err != nil { + t.Fatalf("unexpected error: %v", err) + } + + got := getClusterOrNil(t, c, "mycluster", "default") + if got == nil { + t.Fatal("cluster not found after reconcile") + } + if !controllerutil.ContainsFinalizer(got, controller.Finalizer) { + t.Errorf("expected finalizer %q to still be present", controller.Finalizer) + } +} + +// ── US8.2: Nom du cluster depuis le label ou metadata.name ─────────────────── + +// Scenario: Label cluster.x-k8s.io/cluster-name présent → nom du label utilisé +func TestReconcile_ClusterName_FromLabel(t *testing.T) { + cluster := newCluster("mycluster-openstack", "default", + withFinalizer, + withDeletionTimestamp, + withClusterLabel("mycluster"), + ) + secret := newSecret("cloud-credentials", "default") + + var capturedName string + r, _ := newReconciler(func(_ context.Context, opts openstack.PurgeOptions) error { + capturedName = opts.ClusterName + return nil + }, cluster, secret) + + if _, err := r.Reconcile(context.Background(), reconcileRequest("mycluster-openstack", "default")); err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if capturedName != "mycluster" { + t.Errorf("expected ClusterName %q (from label), got %q", "mycluster", capturedName) + } +} + +// Scenario: Label absent → metadata.name utilisé +func TestReconcile_ClusterName_FallsBackToMetadataName(t *testing.T) { + cluster := newCluster("mycluster", "default", withFinalizer, withDeletionTimestamp) + secret := newSecret("cloud-credentials", "default") + + var capturedName string + r, _ := newReconciler(func(_ context.Context, opts openstack.PurgeOptions) error { + capturedName = opts.ClusterName + return nil + }, cluster, secret) + + if _, err := r.Reconcile(context.Background(), reconcileRequest("mycluster", "default")); err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if capturedName != "mycluster" { + t.Errorf("expected ClusterName %q (from metadata.name), got %q", "mycluster", capturedName) + } +} + +// ── US8.3: Supprimer le finalizer après nettoyage réussi ───────────────────── + +// Scenario: Purge réussie → finalizer retiré +func TestReconcile_RemovesFinalizer_AfterSuccessfulPurge(t *testing.T) { + cluster := newCluster("mycluster", "default", withFinalizer, withDeletionTimestamp) + secret := newSecret("cloud-credentials", "default") + + r, c := newReconciler(func(_ context.Context, _ openstack.PurgeOptions) error { return nil }, cluster, secret) + + if _, err := r.Reconcile(context.Background(), reconcileRequest("mycluster", "default")); err != nil { + t.Fatalf("unexpected error: %v", err) + } + + // After removing the last finalizer with DeletionTimestamp set, the fake client may GC the object. + got := getClusterOrNil(t, c, "mycluster", "default") + if got != nil && controllerutil.ContainsFinalizer(got, controller.Finalizer) { + t.Errorf("expected finalizer %q to be removed after purge", controller.Finalizer) + } +} + +// Scenario: Finalizer absent au moment de la suppression → pas de purge +func TestReconcile_SkipsCleanup_WhenFinalizerAbsent(t *testing.T) { + // Use a non-janitor finalizer to keep the cluster alive in the fake client + // when we call Delete (which sets DeletionTimestamp instead of deleting immediately). + cluster := newCluster("mycluster", "default") + controllerutil.AddFinalizer(cluster, "other.finalizer.example.com") + + purgeCalled := false + r, c := newReconciler(func(_ context.Context, _ openstack.PurgeOptions) error { + purgeCalled = true + return nil + }, cluster) + + // Trigger deletion — fake client sets DeletionTimestamp (kept alive by other finalizer). + if err := c.Delete(context.Background(), cluster); err != nil { + t.Fatalf("marking cluster for deletion: %v", err) + } + + if _, err := r.Reconcile(context.Background(), reconcileRequest("mycluster", "default")); err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if purgeCalled { + t.Error("expected purge NOT to be called when janitor finalizer is absent") + } +} + +// ── US8.4: Mécanisme de retry via annotation ────────────────────────────────── + +// Scenario: Erreur lors de la purge → annotation retry posée, Reconcile retourne nil +func TestReconcile_AnnotatesRetry_OnPurgeError(t *testing.T) { + cluster := newCluster("mycluster", "default", withFinalizer, withDeletionTimestamp) + secret := newSecret("cloud-credentials", "default") + + r, c := newReconciler(func(_ context.Context, _ openstack.PurgeOptions) error { + return errors.New("purge failed") + }, cluster, secret) + + if _, err := r.Reconcile(context.Background(), reconcileRequest("mycluster", "default")); err != nil { + t.Fatalf("expected nil (retry handled internally), got: %v", err) + } + + got := getClusterOrNil(t, c, "mycluster", "default") + if got == nil { + t.Fatal("cluster not found after reconcile") + } + if got.Annotations[controller.RetryAnnotation] == "" { + t.Errorf("expected retry annotation %q to be set after purge error", controller.RetryAnnotation) + } +} + +// Scenario: Cluster supprimé entre l'erreur de purge et l'annotation retry → NotFound ignoré +func TestReconcile_IgnoresNotFound_WhenClusterDeletedDuringRetry(t *testing.T) { + cluster := newCluster("mycluster", "default", withFinalizer, withDeletionTimestamp) + secret := newSecret("cloud-credentials", "default") + + r, c := newReconciler(nil, cluster, secret) + + // PurgeFunc deletes the cluster mid-flight to simulate it disappearing during cleanup. + r.PurgeFunc = func(ctx context.Context, _ openstack.PurgeOptions) error { + // Remove finalizer first so the fake client actually deletes on Delete call. + var cl infrav1.OpenStackCluster + _ = c.Get(ctx, types.NamespacedName{Name: "mycluster", Namespace: "default"}, &cl) + controllerutil.RemoveFinalizer(&cl, controller.Finalizer) + _ = c.Update(ctx, &cl) + _ = c.Delete(ctx, &cl) + return fmt.Errorf("cleanup failed") + } + + // annotateRetry will see NotFound — must be ignored, not propagated. + if _, err := r.Reconcile(context.Background(), reconcileRequest("mycluster", "default")); err != nil { + t.Fatalf("expected no error when cluster deleted during retry annotation, got: %v", err) + } +} From 9f62675c74a4c904b7d5805a2d288424127ad190 Mon Sep 17 00:00:00 2001 From: Mathieu Grzybek Date: Wed, 8 Jul 2026 09:57:40 +0200 Subject: [PATCH 10/27] =?UTF-8?q?feat(controller):=20Epic=209=20=E2=80=94?= =?UTF-8?q?=20Configuration=20via=20env=20vars=20(TDD)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - config_test.go: 10 tests covering US9.1 — DefaultVolumesFromEnv (default "delete", reads "keep", reads "delete"), RetryDelayFromEnv (default 60, reads 120, invalid value → fallback 60), reconciler volumes policy (global delete/keep, annotation overrides in both directions), retry delay applied to SleepFunc Signed-off-by: Mathieu Grzybek --- internal/controller/config_test.go | 146 +++++++++++++++++++++++++++++ 1 file changed, 146 insertions(+) create mode 100644 internal/controller/config_test.go diff --git a/internal/controller/config_test.go b/internal/controller/config_test.go new file mode 100644 index 0000000..ab58434 --- /dev/null +++ b/internal/controller/config_test.go @@ -0,0 +1,146 @@ +package controller_test + +import ( + "context" + "testing" + "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + infrav1 "sigs.k8s.io/cluster-api-provider-openstack/api/v1beta1" + + "github.com/azimuth-cloud/cluster-api-janitor-openstack/internal/controller" + "github.com/azimuth-cloud/cluster-api-janitor-openstack/internal/openstack" +) + +// ── US9.1: Configuration via variables d'environnement ──────────────────────── + +// Scenario: CAPI_JANITOR_DEFAULT_VOLUMES_POLICY non définie → "delete" +func TestDefaultVolumesFromEnv_DefaultsToDelete(t *testing.T) { + t.Setenv("CAPI_JANITOR_DEFAULT_VOLUMES_POLICY", "") + if got := controller.DefaultVolumesFromEnv(); got != controller.PolicyDelete { + t.Errorf("expected %q, got %q", controller.PolicyDelete, got) + } +} + +// Scenario: CAPI_JANITOR_DEFAULT_VOLUMES_POLICY = "keep" → "keep" +func TestDefaultVolumesFromEnv_ReadsKeep(t *testing.T) { + t.Setenv("CAPI_JANITOR_DEFAULT_VOLUMES_POLICY", "keep") + if got := controller.DefaultVolumesFromEnv(); got != "keep" { + t.Errorf("expected %q, got %q", "keep", got) + } +} + +// Scenario: CAPI_JANITOR_RETRY_DEFAULT_DELAY non définie → 60s (défaut) +func TestRetryDelayFromEnv_DefaultsTo60(t *testing.T) { + t.Setenv("CAPI_JANITOR_RETRY_DEFAULT_DELAY", "") + if got := controller.RetryDelayFromEnv(); got != 60 { + t.Errorf("expected 60, got %d", got) + } +} + +// Scenario: CAPI_JANITOR_RETRY_DEFAULT_DELAY = "120" → 120 +func TestRetryDelayFromEnv_ReadsConfiguredDelay(t *testing.T) { + t.Setenv("CAPI_JANITOR_RETRY_DEFAULT_DELAY", "120") + if got := controller.RetryDelayFromEnv(); got != 120 { + t.Errorf("expected 120, got %d", got) + } +} + +// Scenario: CAPI_JANITOR_RETRY_DEFAULT_DELAY invalide → 60 (fallback) +func TestRetryDelayFromEnv_InvalidValue_FallsBackToDefault(t *testing.T) { + t.Setenv("CAPI_JANITOR_RETRY_DEFAULT_DELAY", "not-a-number") + if got := controller.RetryDelayFromEnv(); got != 60 { + t.Errorf("expected 60 for invalid value, got %d", got) + } +} + +// ── Politique volumes via le reconciler ─────────────────────────────────────── + +func withVolumePolicy(policy string) func(*infrav1.OpenStackCluster) { + return func(c *infrav1.OpenStackCluster) { + if c.Annotations == nil { + c.Annotations = make(map[string]string) + } + c.Annotations[controller.VolumesPolicyAnnotation] = policy + } +} + +func reconcileForVolumesCapture(t *testing.T, defaultPolicy string, clusterOpts ...func(*infrav1.OpenStackCluster)) openstack.PurgeOptions { + t.Helper() + opts := append([]func(*infrav1.OpenStackCluster){withFinalizer, withDeletionTimestamp}, clusterOpts...) + cluster := newCluster("mycluster", "default", opts...) + secret := newSecret("cloud-credentials", "default") + + var captured openstack.PurgeOptions + r, _ := newReconciler(func(_ context.Context, o openstack.PurgeOptions) error { + captured = o + return nil + }, cluster, secret) + r.DefaultVolumesPolicy = defaultPolicy + + if _, err := r.Reconcile(context.Background(), reconcileRequest("mycluster", "default")); err != nil { + t.Fatalf("unexpected error: %v", err) + } + return captured +} + +// Scenario: Politique globale "delete" → volumes inclus dans la purge +func TestReconcile_VolumesPolicy_IncludesVolumes_WhenDelete(t *testing.T) { + opts := reconcileForVolumesCapture(t, controller.PolicyDelete) + if !opts.IncludeVolumes { + t.Error("expected IncludeVolumes=true for policy 'delete'") + } +} + +// Scenario: Politique globale "keep" → volumes exclus de la purge +func TestReconcile_VolumesPolicy_ExcludesVolumes_WhenKeep(t *testing.T) { + opts := reconcileForVolumesCapture(t, "keep") + if opts.IncludeVolumes { + t.Error("expected IncludeVolumes=false for policy 'keep'") + } +} + +// Scenario: Annotation "delete" sur le cluster (override keep global) +func TestReconcile_VolumesPolicy_AnnotationDeleteOverridesKeepGlobal(t *testing.T) { + opts := reconcileForVolumesCapture(t, "keep", withVolumePolicy(controller.PolicyDelete)) + if !opts.IncludeVolumes { + t.Error("expected IncludeVolumes=true when annotation 'delete' overrides global 'keep'") + } +} + +// Scenario: Annotation "keep" sur le cluster (override delete global) +func TestReconcile_VolumesPolicy_AnnotationKeepOverridesDeleteGlobal(t *testing.T) { + opts := reconcileForVolumesCapture(t, controller.PolicyDelete, withVolumePolicy("keep")) + if opts.IncludeVolumes { + t.Error("expected IncludeVolumes=false when annotation 'keep' overrides global 'delete'") + } +} + +// ── Délai de retry configurable ─────────────────────────────────────────────── + +// Scenario: RetryDefaultDelay = 120 → sleep appelé avec 120 secondes +func TestReconcile_RetryDelay_UsesConfiguredDelay(t *testing.T) { + cluster := newCluster("mycluster", "default", + withFinalizer, + func(c *infrav1.OpenStackCluster) { + now := metav1.Now() + c.DeletionTimestamp = &now + }, + ) + secret := newSecret("cloud-credentials", "default") + + var sleptFor time.Duration + r, _ := newReconciler(func(_ context.Context, _ openstack.PurgeOptions) error { + return context.DeadlineExceeded // simulate purge failure + }, cluster, secret) + r.RetryDefaultDelay = 120 + r.SleepFunc = func(d time.Duration) { sleptFor = d } + + if _, err := r.Reconcile(context.Background(), reconcileRequest("mycluster", "default")); err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if sleptFor != 120*time.Second { + t.Errorf("expected sleep of 120s, got %v", sleptFor) + } +} From 9ed6c538a52f908f43cdc33905ddb96044cfa268 Mon Sep 17 00:00:00 2001 From: Mathieu Grzybek Date: Wed, 8 Jul 2026 11:21:15 +0200 Subject: [PATCH 11/27] =?UTF-8?q?feat(packaging):=20Epic=2010=20=E2=80=94?= =?UTF-8?q?=20OCI=20image,=20Helm=20chart=20and=20SBOM=20via=20Nix=20(TDD)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit US10.1 — Secure image - Dockerfile: full rewrite from Python/kopf to Go multi-stage (golang:1.26 builder + gcr.io/distroless/static:nonroot, UID 65532) US10.2 — Helm chart - chart/values.yaml: add retryDefaultDelay (default 60 s) - chart/templates/deployment.yaml: env CAPI_JANITOR_RETRY_DEFAULT_DELAY, livenessProbe (/healthz:8081) and readinessProbe (/readyz:8081) - chart/templates/clusterrole.yaml: add "update" verb on openstackclusters (required by r.Update for finalizer management); remove stale kopf comments - chart/tests/packaging_test.yaml: 11 helm-unittest tests covering US10.1 and US10.2 (security context, injectable env vars, probes, RBAC) US10.3 — Nix-based OCI build (no Flake) + SBOM - nix/nixpkgs.nix: pin nixos-26.05 (Go 1.26 required) - nix/default.nix: four attributes — manager (buildGoModule, CGO_ENABLED=0), amd64 image, arm64 image (cross-compiled via pkgsCross.aarch64-multiplatform), sbom (syft scan of the binary → CycloneDX JSON) - .github/workflows/build-push-artifacts.yaml: replace docker-multiarch-build-push with nix-build + skopeo push + docker manifest for multi-arch support; SBOM generation uploaded as a CI artifact Note: vendorHash = lib.fakeHash — update on first nix-build run. Signed-off-by: Mathieu Grzybek --- .github/workflows/build-push-artifacts.yaml | 79 +++++++++++---- Dockerfile | 57 +++-------- chart/templates/clusterrole.yaml | 5 +- chart/templates/deployment.yaml | 14 +++ chart/tests/packaging_test.yaml | 103 ++++++++++++++++++++ chart/values.yaml | 3 + nix/default.nix | 54 ++++++++++ nix/nixpkgs.nix | 5 + 8 files changed, 254 insertions(+), 66 deletions(-) create mode 100644 chart/tests/packaging_test.yaml create mode 100644 nix/default.nix create mode 100644 nix/nixpkgs.nix diff --git a/.github/workflows/build-push-artifacts.yaml b/.github/workflows/build-push-artifacts.yaml index 9687e84..e3af823 100644 --- a/.github/workflows/build-push-artifacts.yaml +++ b/.github/workflows/build-push-artifacts.yaml @@ -23,7 +23,7 @@ on: jobs: build_push_images: - name: Build and push images + name: Build and push images (Nix) runs-on: ubuntu-latest permissions: contents: read @@ -36,6 +36,9 @@ jobs: with: ref: ${{ inputs.ref }} + - name: Install Nix + uses: DeterminateSystems/nix-installer-action@07ebb8d2749aa2fac2baae7d1cfa011e4acfd8d1 # v5 + - name: Login to GitHub Container Registry uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4.0.0 with: @@ -47,27 +50,63 @@ jobs: id: semver uses: azimuth-cloud/github-actions/semver@master - - name: Calculate metadata for image - id: image-meta - uses: docker/metadata-action@030e881283bb7a6894de51c315a6bfe6a94e05cf # v6.0.0 - with: - images: ghcr.io/azimuth-cloud/cluster-api-janitor-openstack - # Produce the branch name or tag and the SHA as tags - tags: | - type=ref,event=branch - type=ref,event=tag - type=raw,value=${{ steps.semver.outputs.short-sha }} + - name: Compute image tags + id: tags + run: | + # All final tags for the manifest list (space-separated) + TAGS="" + REF="${{ github.ref }}" + SHA="${{ steps.semver.outputs.short-sha }}" + IMAGE="ghcr.io/azimuth-cloud/cluster-api-janitor-openstack" + if [[ "$REF" == refs/tags/* ]]; then + TAG="${REF#refs/tags/}" + TAGS="$IMAGE:$TAG $IMAGE:$SHA" + elif [[ "$REF" == refs/heads/* ]]; then + BRANCH="${REF#refs/heads/}" + TAGS="$IMAGE:$BRANCH $IMAGE:$SHA" + else + TAGS="$IMAGE:$SHA" + fi + echo "tags=$TAGS" >> "$GITHUB_OUTPUT" + echo "sha_tag=$IMAGE:$SHA" >> "$GITHUB_OUTPUT" + echo "image=$IMAGE" >> "$GITHUB_OUTPUT" + + - name: Build amd64 image + run: nix-build nix -A image -o image-amd64.tar.gz + + - name: Build arm64 image (cross-compiled) + run: nix-build nix -A image-arm64 -o image-arm64.tar.gz + + - name: Push arch-specific images and create manifest list + env: + SHA_TAG: ${{ steps.tags.outputs.sha_tag }} + IMAGE: ${{ steps.tags.outputs.image }} + TAGS: ${{ steps.tags.outputs.tags }} + run: | + # Push each arch image under a unique tag used only to assemble the manifest. + skopeo copy \ + docker-archive:image-amd64.tar.gz \ + "docker://${SHA_TAG}-amd64" + skopeo copy \ + docker-archive:image-arm64.tar.gz \ + "docker://${SHA_TAG}-arm64" + + # For each final tag, create and push a multi-arch manifest list. + for TAG in $TAGS; do + docker manifest create "$TAG" \ + --amend "${SHA_TAG}-amd64" \ + --amend "${SHA_TAG}-arm64" + docker manifest push "$TAG" + done + + - name: Generate SBOM (CycloneDX) + run: nix-build nix -A sbom -o sbom.cdx.json - - name: Build and push image - uses: azimuth-cloud/github-actions/docker-multiarch-build-push@master + - name: Upload SBOM as artifact + uses: actions/upload-artifact@v4 with: - cache-key: cluster-api-janitor-openstack - context: . - platforms: linux/amd64,linux/arm64 - push: true - tags: ${{ steps.image-meta.outputs.tags }} - labels: ${{ steps.image-meta.outputs.labels }} - github-token: ${{ secrets.GITHUB_TOKEN }} + name: sbom-${{ steps.semver.outputs.short-sha }} + path: sbom.cdx.json build_push_chart: name: Build and push Helm chart diff --git a/Dockerfile b/Dockerfile index 8e3c729..3dd2247 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,42 +1,15 @@ -FROM ubuntu:24.04 AS python-builder - -RUN apt-get update && \ - apt-get install -y python3 python3-venv - -RUN python3 -m venv /venv && \ - /venv/bin/pip install -U pip setuptools - -COPY requirements.txt /app/requirements.txt -RUN /venv/bin/pip install --requirement /app/requirements.txt - -COPY . /app -RUN /venv/bin/pip install /app - - -FROM ubuntu:24.04 - -# Don't buffer stdout and stderr as it breaks realtime logging -ENV PYTHONUNBUFFERED=1 - -# Create the user that will be used to run the app -ENV APP_UID=1001 -ENV APP_GID=1001 -ENV APP_USER=app -ENV APP_GROUP=app -RUN groupadd --gid $APP_GID $APP_GROUP && \ - useradd \ - --no-create-home \ - --no-user-group \ - --gid $APP_GID \ - --shell /sbin/nologin \ - --uid $APP_UID \ - $APP_USER - -RUN apt-get update && \ - apt-get install -y ca-certificates python3 tini && \ - rm -rf /var/lib/apt/lists/* - -COPY --from=python-builder /venv /venv - -USER $APP_UID -CMD ["/venv/bin/kopf", "run", "--module", "capi_janitor.openstack.operator", "--all-namespaces", "--verbose"] +FROM golang:1.26 AS builder +ARG TARGETOS +ARG TARGETARCH +WORKDIR /workspace +COPY go.mod go.sum ./ +RUN go mod download +COPY . . +RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} \ + go build -a -o manager ./cmd/main.go + +FROM gcr.io/distroless/static:nonroot +WORKDIR / +COPY --from=builder /workspace/manager . +USER 65532:65532 +ENTRYPOINT ["/manager"] diff --git a/chart/templates/clusterrole.yaml b/chart/templates/clusterrole.yaml index b54b30b..759adb9 100644 --- a/chart/templates/clusterrole.yaml +++ b/chart/templates/clusterrole.yaml @@ -4,7 +4,6 @@ metadata: name: {{ include "cluster-api-janitor-openstack.fullname" . }} labels: {{ include "cluster-api-janitor-openstack.labels" . | nindent 4 }} rules: - # Required for kopf to watch resources properly - apiGroups: - "" resources: @@ -12,7 +11,6 @@ rules: verbs: - list - watch - # Required for kopf to produce events properly - apiGroups: - "" - events.k8s.io @@ -20,7 +18,6 @@ rules: - events verbs: - create - # We need access to OpenStackClusters and the cloud credential secrets - apiGroups: - "" resources: @@ -37,7 +34,7 @@ rules: - get - watch - patch - # Required to prevent erroneous error logs during kopf startup + - update - apiGroups: - apiextensions.k8s.io resources: diff --git a/chart/templates/deployment.yaml b/chart/templates/deployment.yaml index 9e84cc1..a7a18f8 100644 --- a/chart/templates/deployment.yaml +++ b/chart/templates/deployment.yaml @@ -27,6 +27,20 @@ spec: env: - name: CAPI_JANITOR_DEFAULT_VOLUMES_POLICY value: {{ .Values.defaultVolumesPolicy }} + - name: CAPI_JANITOR_RETRY_DEFAULT_DELAY + value: {{ .Values.retryDefaultDelay | quote }} + livenessProbe: + httpGet: + path: /healthz + port: 8081 + initialDelaySeconds: 15 + periodSeconds: 20 + readinessProbe: + httpGet: + path: /readyz + port: 8081 + initialDelaySeconds: 5 + periodSeconds: 10 resources: {{ toYaml .Values.resources | nindent 12 }} volumeMounts: - name: tmp diff --git a/chart/tests/packaging_test.yaml b/chart/tests/packaging_test.yaml new file mode 100644 index 0000000..0614395 --- /dev/null +++ b/chart/tests/packaging_test.yaml @@ -0,0 +1,103 @@ +# Epic 10 — US10.1 / US10.2 packaging scenarios +# Run with: docker run -i --rm -v $(pwd):/apps helmunittest/helm-unittest chart +suite: Packaging and deployment tests + +tests: + # US10.1 — Image sécurisée + - it: should run pods as non-root (US10.1) + template: templates/deployment.yaml + asserts: + - equal: + path: spec.template.spec.securityContext.runAsNonRoot + value: true + + - it: should use a read-only root filesystem (US10.1) + template: templates/deployment.yaml + asserts: + - equal: + path: spec.template.spec.containers[0].securityContext.readOnlyRootFilesystem + value: true + + - it: should drop all Linux capabilities (US10.1) + template: templates/deployment.yaml + asserts: + - contains: + path: spec.template.spec.containers[0].securityContext.capabilities.drop + content: ALL + + # US10.2 — Helm chart + - it: should create four resources (US10.2) + asserts: + - hasDocuments: + count: 4 + + - it: should inject CAPI_JANITOR_DEFAULT_VOLUMES_POLICY with default value "delete" (US10.2) + template: templates/deployment.yaml + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: CAPI_JANITOR_DEFAULT_VOLUMES_POLICY + value: delete + + - it: should inject overridden defaultVolumesPolicy "keep" (US10.2) + template: templates/deployment.yaml + set: + defaultVolumesPolicy: keep + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: CAPI_JANITOR_DEFAULT_VOLUMES_POLICY + value: keep + + - it: should inject CAPI_JANITOR_RETRY_DEFAULT_DELAY with default value "60" (US10.2) + template: templates/deployment.yaml + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: CAPI_JANITOR_RETRY_DEFAULT_DELAY + value: "60" + + - it: should inject overridden retryDefaultDelay (US10.2) + template: templates/deployment.yaml + set: + retryDefaultDelay: 120 + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: CAPI_JANITOR_RETRY_DEFAULT_DELAY + value: "120" + + - it: should have a liveness probe at /healthz (US10.2) + template: templates/deployment.yaml + asserts: + - equal: + path: spec.template.spec.containers[0].livenessProbe.httpGet.path + value: /healthz + + - it: should have a readiness probe at /readyz (US10.2) + template: templates/deployment.yaml + asserts: + - equal: + path: spec.template.spec.containers[0].readinessProbe.httpGet.path + value: /readyz + + - it: should grant update verb on openstackclusters (US10.2) + template: templates/clusterrole.yaml + asserts: + - contains: + path: rules + content: + apiGroups: + - infrastructure.cluster.x-k8s.io + resources: + - openstackclusters + verbs: + - list + - get + - watch + - patch + - update diff --git a/chart/values.yaml b/chart/values.yaml index bcb4d49..238092c 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -1,6 +1,9 @@ # The default volumes policy for the operator defaultVolumesPolicy: delete +# The default retry delay in seconds after a failed cleanup attempt +retryDefaultDelay: 60 + # The image to use for the operator image: repository: ghcr.io/azimuth-cloud/cluster-api-janitor-openstack diff --git a/nix/default.nix b/nix/default.nix new file mode 100644 index 0000000..99ca6d5 --- /dev/null +++ b/nix/default.nix @@ -0,0 +1,54 @@ +{ pkgs ? import ./nixpkgs.nix }: + +let + # Build the manager binary for a given package set (native or cross). + buildManager = p: p.buildGoModule { + pname = "cluster-api-janitor-openstack"; + version = "0.0.0-dev"; + src = ../.; + subPackages = [ "cmd" ]; + CGO_ENABLED = "0"; + ldflags = [ "-s" "-w" ]; + # Run `nix-build nix -A manager` once; it will fail and print the real hash. + vendorHash = pkgs.lib.fakeHash; + postInstall = '' + mv $out/bin/cmd $out/bin/manager + ''; + meta.mainProgram = "manager"; + }; + + # Build a layered OCI image for a given package set. + buildImage = p: m: p.dockerTools.buildLayeredImage { + name = "ghcr.io/azimuth-cloud/cluster-api-janitor-openstack"; + tag = "latest"; + contents = [ pkgs.cacert m ]; + config = { + Entrypoint = [ "/bin/manager" ]; + ExposedPorts."8081/tcp" = {}; + User = "65532:65532"; + Labels = { + "org.opencontainers.image.source" = + "https://github.com/azimuth-cloud/cluster-api-janitor-openstack"; + "org.opencontainers.image.licenses" = "Apache-2.0"; + }; + }; + }; + + manager = buildManager pkgs; + image = buildImage pkgs manager; + + # arm64 cross-compiled on an amd64 host. + crossPkgs = pkgs.pkgsCross.aarch64-multiplatform; + manager-arm64 = buildManager crossPkgs; + image-arm64 = buildImage crossPkgs manager-arm64; + + # SBOM — reads Go build-info embedded in the static binary (survives -s -w). + sbom = pkgs.runCommand "sbom.cdx.json" { + nativeBuildInputs = [ pkgs.syft ]; + } '' + syft scan ${manager}/bin/manager \ + --output cyclonedx-json \ + --file $out + ''; + +in { inherit manager image image-arm64 sbom; } diff --git a/nix/nixpkgs.nix b/nix/nixpkgs.nix new file mode 100644 index 0000000..45a564c --- /dev/null +++ b/nix/nixpkgs.nix @@ -0,0 +1,5 @@ +import (builtins.fetchTarball { + # nixos-26.05 — require Go 1.26+ + # To update: nix-prefetch-url --unpack https://github.com/NixOS/nixpkgs/archive/.tar.gz + url = "https://github.com/NixOS/nixpkgs/archive/refs/heads/nixos-26.05.tar.gz"; +}) {} From cfccd2ebe18ee66dc4c5ebefb4907075179f438c Mon Sep 17 00:00:00 2001 From: Mathieu Grzybek Date: Wed, 8 Jul 2026 11:29:44 +0200 Subject: [PATCH 12/27] =?UTF-8?q?feat(controller):=20Epic=2011=20=E2=80=94?= =?UTF-8?q?=20Observability=20(Prometheus=20metrics=20+=20Kubernetes=20eve?= =?UTF-8?q?nts)=20(TDD)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit US11.1 — Prometheus metrics - internal/controller/metrics.go: Metrics struct exposing capi_janitor_cleanups_total{result="success|failure"} CounterVec; NewMetrics(reg) registers it against any prometheus.Registerer - OpenStackClusterReconciler.Metrics field (optional, injected in tests); SetupWithManager auto-registers against ctrlmetrics.Registry when nil - incMetric("success"|"failure") called after each purge attempt US11.2 — Kubernetes events - OpenStackClusterReconciler.Recorder field (record.EventRecorder); SetupWithManager wires mgr.GetEventRecorderFor("capi-janitor") when nil - Normal/CleanupSucceeded emitted on successful OpenStack cleanup - Warning/CleanupFailed emitted on purge error (message carries the error) Tests - internal/controller/observability_test.go: 4 TDD tests using prometheus.NewRegistry() + prometheus/testutil.ToFloat64 for metric assertions, record.NewFakeRecorder for event assertions go.mod: prometheus/client_golang promoted from indirect to direct dependency Signed-off-by: Mathieu Grzybek --- go.mod | 3 +- internal/controller/metrics.go | 21 +++ internal/controller/observability_test.go | 122 ++++++++++++++++++ .../controller/openstackcluster_controller.go | 27 ++++ 4 files changed, 172 insertions(+), 1 deletion(-) create mode 100644 internal/controller/metrics.go create mode 100644 internal/controller/observability_test.go diff --git a/go.mod b/go.mod index 1702953..52079b8 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ require ( github.com/go-logr/logr v1.4.3 github.com/onsi/ginkgo/v2 v2.28.2 github.com/onsi/gomega v1.42.0 + github.com/prometheus/client_golang v1.23.2 k8s.io/api v0.36.0 k8s.io/apimachinery v0.36.0 k8s.io/client-go v0.36.0 @@ -44,13 +45,13 @@ require ( github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect + github.com/kylelemons/godebug v1.1.0 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect - github.com/prometheus/client_golang v1.23.2 // indirect github.com/prometheus/client_model v0.6.2 // indirect github.com/prometheus/common v0.67.5 // indirect github.com/prometheus/procfs v0.19.2 // indirect diff --git a/internal/controller/metrics.go b/internal/controller/metrics.go new file mode 100644 index 0000000..f7a3d91 --- /dev/null +++ b/internal/controller/metrics.go @@ -0,0 +1,21 @@ +package controller + +import "github.com/prometheus/client_golang/prometheus" + +// Metrics holds the Prometheus counters for the janitor controller. +type Metrics struct { + CleanupsTotal *prometheus.CounterVec +} + +// NewMetrics creates and registers the janitor metrics with reg. +func NewMetrics(reg prometheus.Registerer) *Metrics { + m := &Metrics{ + CleanupsTotal: prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: "capi_janitor", + Name: "cleanups_total", + Help: "Total number of OpenStack cluster cleanups, labelled by result (success|failure).", + }, []string{"result"}), + } + reg.MustRegister(m.CleanupsTotal) + return m +} diff --git a/internal/controller/observability_test.go b/internal/controller/observability_test.go new file mode 100644 index 0000000..54f0dea --- /dev/null +++ b/internal/controller/observability_test.go @@ -0,0 +1,122 @@ +package controller_test + +import ( + "context" + "errors" + "strings" + "testing" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/testutil" + "k8s.io/client-go/tools/record" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/azimuth-cloud/cluster-api-janitor-openstack/internal/controller" + "github.com/azimuth-cloud/cluster-api-janitor-openstack/internal/openstack" +) + +func newObsReconciler( + reg *prometheus.Registry, + purgeFunc func(context.Context, openstack.PurgeOptions) error, + objs ...client.Object, +) (*controller.OpenStackClusterReconciler, *record.FakeRecorder) { + r, _ := newReconciler(purgeFunc, objs...) + r.Metrics = controller.NewMetrics(reg) + rec := record.NewFakeRecorder(10) + r.Recorder = rec + return r, rec +} + +// ── US11.1 : Métriques Prometheus ──────────────────────────────────────────── + +// Scenario: nettoyage réussi → capi_janitor_cleanups_total{result="success"} += 1 +func TestMetrics_IncrementsSuccess_OnCleanup(t *testing.T) { + cluster := newCluster("c", "default", withFinalizer, withDeletionTimestamp) + secret := newSecret("cloud-credentials", "default") + + reg := prometheus.NewRegistry() + r, _ := newObsReconciler(reg, + func(_ context.Context, _ openstack.PurgeOptions) error { return nil }, + cluster, secret, + ) + + if _, err := r.Reconcile(context.Background(), reconcileRequest("c", "default")); err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if got := testutil.ToFloat64(r.Metrics.CleanupsTotal.WithLabelValues("success")); got != 1 { + t.Errorf("expected cleanupsTotal{result=success}=1, got %v", got) + } +} + +// Scenario: purge échouée → capi_janitor_cleanups_total{result="failure"} += 1 +func TestMetrics_IncrementsFailure_OnPurgeError(t *testing.T) { + cluster := newCluster("c", "default", withFinalizer, withDeletionTimestamp) + secret := newSecret("cloud-credentials", "default") + + reg := prometheus.NewRegistry() + r, _ := newObsReconciler(reg, + func(_ context.Context, _ openstack.PurgeOptions) error { return errors.New("purge failed") }, + cluster, secret, + ) + + if _, err := r.Reconcile(context.Background(), reconcileRequest("c", "default")); err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if got := testutil.ToFloat64(r.Metrics.CleanupsTotal.WithLabelValues("failure")); got != 1 { + t.Errorf("expected cleanupsTotal{result=failure}=1, got %v", got) + } +} + +// ── US11.2 : Événements Kubernetes ─────────────────────────────────────────── + +// Scenario: nettoyage réussi → événement Normal "CleanupSucceeded" +func TestEvents_EmitsNormal_OnCleanupSuccess(t *testing.T) { + cluster := newCluster("c", "default", withFinalizer, withDeletionTimestamp) + secret := newSecret("cloud-credentials", "default") + + reg := prometheus.NewRegistry() + r, rec := newObsReconciler(reg, + func(_ context.Context, _ openstack.PurgeOptions) error { return nil }, + cluster, secret, + ) + + if _, err := r.Reconcile(context.Background(), reconcileRequest("c", "default")); err != nil { + t.Fatalf("unexpected error: %v", err) + } + + select { + case event := <-rec.Events: + if !strings.Contains(event, "Normal") || !strings.Contains(event, "CleanupSucceeded") { + t.Errorf("expected Normal/CleanupSucceeded event, got: %q", event) + } + default: + t.Error("no event was recorded") + } +} + +// Scenario: purge échouée → événement Warning "CleanupFailed" +func TestEvents_EmitsWarning_OnPurgeFailure(t *testing.T) { + cluster := newCluster("c", "default", withFinalizer, withDeletionTimestamp) + secret := newSecret("cloud-credentials", "default") + + reg := prometheus.NewRegistry() + r, rec := newObsReconciler(reg, + func(_ context.Context, _ openstack.PurgeOptions) error { return errors.New("purge failed") }, + cluster, secret, + ) + + if _, err := r.Reconcile(context.Background(), reconcileRequest("c", "default")); err != nil { + t.Fatalf("unexpected error: %v", err) + } + + select { + case event := <-rec.Events: + if !strings.Contains(event, "Warning") || !strings.Contains(event, "CleanupFailed") { + t.Errorf("expected Warning/CleanupFailed event, got: %q", event) + } + default: + t.Error("no event was recorded") + } +} diff --git a/internal/controller/openstackcluster_controller.go b/internal/controller/openstackcluster_controller.go index 6ef1503..d190295 100644 --- a/internal/controller/openstackcluster_controller.go +++ b/internal/controller/openstackcluster_controller.go @@ -29,10 +29,12 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/tools/record" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/log" + ctrlmetrics "sigs.k8s.io/controller-runtime/pkg/metrics" infrav1 "sigs.k8s.io/cluster-api-provider-openstack/api/v1beta1" "github.com/azimuth-cloud/cluster-api-janitor-openstack/internal/openstack" @@ -57,6 +59,8 @@ type OpenStackClusterReconciler struct { Scheme *runtime.Scheme DefaultVolumesPolicy string RetryDefaultDelay int + Metrics *Metrics + Recorder record.EventRecorder // PurgeFunc is called to clean up OpenStack resources; defaults to openstack.PurgeResources. PurgeFunc func(context.Context, openstack.PurgeOptions) error // SleepFunc is called instead of time.Sleep; defaults to time.Sleep. @@ -78,6 +82,18 @@ func (r *OpenStackClusterReconciler) sleep(d time.Duration) { } } +func (r *OpenStackClusterReconciler) incMetric(result string) { + if r.Metrics != nil { + r.Metrics.CleanupsTotal.WithLabelValues(result).Inc() + } +} + +func (r *OpenStackClusterReconciler) recordEvent(obj client.Object, eventType, reason, msg string) { + if r.Recorder != nil { + r.Recorder.Event(obj, eventType, reason, msg) + } +} + //+kubebuilder:rbac:groups=infrastructure.cluster.x-k8s.io,resources=openstackclusters,verbs=get;list;watch;patch;update //+kubebuilder:rbac:groups="",resources=secrets,verbs=get;delete //+kubebuilder:rbac:groups="",resources=namespaces,verbs=list;watch @@ -147,6 +163,8 @@ func (r *OpenStackClusterReconciler) Reconcile(ctx context.Context, req ctrl.Req Logger: logger, }) if purgeErr != nil { + r.incMetric("failure") + r.recordEvent(&cluster, corev1.EventTypeWarning, "CleanupFailed", purgeErr.Error()) logger.Error(purgeErr, "purge failed, will retry") r.sleep(r.retryDelay()) if err := r.annotateRetry(ctx, &cluster); err != nil && !apierrors.IsNotFound(err) { @@ -155,6 +173,9 @@ func (r *OpenStackClusterReconciler) Reconcile(ctx context.Context, req ctrl.Req return ctrl.Result{}, nil } + r.incMetric("success") + r.recordEvent(&cluster, corev1.EventTypeNormal, "CleanupSucceeded", "OpenStack resources cleaned up successfully") + // Delete appcred secret if this is the last finalizer and policy says so. if credentialPolicy == PolicyDelete { if len(cluster.Finalizers) == 1 { @@ -185,6 +206,12 @@ func (r *OpenStackClusterReconciler) Reconcile(ctx context.Context, req ctrl.Req // SetupWithManager registers the reconciler with the controller manager. func (r *OpenStackClusterReconciler) SetupWithManager(mgr ctrl.Manager) error { + if r.Recorder == nil { + r.Recorder = mgr.GetEventRecorderFor("capi-janitor") + } + if r.Metrics == nil { + r.Metrics = NewMetrics(ctrlmetrics.Registry) + } return ctrl.NewControllerManagedBy(mgr). For(&infrav1.OpenStackCluster{}). Complete(r) From 8661e979b4a432607ea762a3378c5c53f802bfb8 Mon Sep 17 00:00:00 2001 From: Mathieu Grzybek Date: Wed, 8 Jul 2026 13:18:45 +0200 Subject: [PATCH 13/27] =?UTF-8?q?feat(openstack):=20Epic=2012=20=E2=80=94?= =?UTF-8?q?=20Robustness:=20HTTP=20timeout=20and=20Cinder=20alias=20(TDD)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit US12.1 — HTTP client timeout - cloud.go: add http.Client.Timeout = 30s as a safety net for callers that do not set a context deadline; context cancellation remains the primary mechanism (all requests already use http.NewRequestWithContext) US12.2 — Cinder legacy alias - resources.go: cinderEndpoint() now tries "volumev3", "block-storage", then "volume" — adds compatibility with OpenStack installations older than Stein that advertise Cinder under the "volume" service type Tests - internal/openstack/robustness_test.go: 3 TDD tests - TestAuthenticate_ReturnsError_WhenContextAlreadyCancelled (US12.1) - TestCinderEndpoint_FallsBackToBlockStorage (regression) - TestCinderEndpoint_FallsBackToVolumeAlias (US12.2) Signed-off-by: Mathieu Grzybek --- internal/openstack/cloud.go | 6 +- internal/openstack/resources.go | 2 +- internal/openstack/robustness_test.go | 154 ++++++++++++++++++++++++++ 3 files changed, 160 insertions(+), 2 deletions(-) create mode 100644 internal/openstack/robustness_test.go diff --git a/internal/openstack/cloud.go b/internal/openstack/cloud.go index f778fc9..639b4e5 100644 --- a/internal/openstack/cloud.go +++ b/internal/openstack/cloud.go @@ -11,6 +11,7 @@ import ( "io" "net/http" "regexp" + "time" "strings" "sigs.k8s.io/yaml" @@ -124,7 +125,10 @@ func Authenticate(ctx context.Context, cloudsYAML, cloudName, cacert string) (*S return nil, err } } - hc := &http.Client{Transport: &http.Transport{TLSClientConfig: tlsCfg}} + hc := &http.Client{ + Transport: &http.Transport{TLSClientConfig: tlsCfg}, + Timeout: 30 * time.Second, + } iface := entry.Interface if iface == "" { diff --git a/internal/openstack/resources.go b/internal/openstack/resources.go index 78c98d9..268b1ca 100644 --- a/internal/openstack/resources.go +++ b/internal/openstack/resources.go @@ -279,7 +279,7 @@ func (s *Session) DeleteAppCredential(ctx context.Context, logger logr.Logger, c } func (s *Session) cinderEndpoint() (string, error) { - return s.endpointFor("volumev3", "block-storage") + return s.endpointFor("volumev3", "block-storage", "volume") } func (s *Session) listVolumeItems(ctx context.Context, endpoint, key string) ([]volumeItem, error) { diff --git a/internal/openstack/robustness_test.go b/internal/openstack/robustness_test.go new file mode 100644 index 0000000..7e3034e --- /dev/null +++ b/internal/openstack/robustness_test.go @@ -0,0 +1,154 @@ +package openstack_test + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "sync" + "testing" + + "github.com/go-logr/logr" + + "github.com/azimuth-cloud/cluster-api-janitor-openstack/internal/openstack" +) + +// ── US12.1 : Timeout HTTP ───────────────────────────────────────────────────── + +// Scenario: le contexte est déjà annulé → Authenticate retourne une erreur immédiatement +func TestAuthenticate_ReturnsError_WhenContextAlreadyCancelled(t *testing.T) { + // Use a real (if short-lived) server so Authenticate reaches the HTTP call. + ks := newKeystoneServer(t) + + ctx, cancel := context.WithCancel(context.Background()) + cancel() // cancel before the call + + cloudsYAML := fmt.Sprintf(` +clouds: + openstack: + auth_type: v3applicationcredential + auth: + auth_url: %s + application_credential_id: cred-id + application_credential_secret: cred-secret +`, ks.URL) + + _, err := openstack.Authenticate(ctx, cloudsYAML, "openstack", "") + if err == nil { + t.Fatal("expected error for cancelled context, got nil") + } +} + +// ── US12.2 : Alias Cinder ──────────────────────────────────────────────────── + +// cinderAliasServer is a minimal Cinder mock that advertises a configurable +// service type in the catalog (allows testing "volume", "block-storage", etc.). +type cinderAliasServer struct { + *httptest.Server + mu sync.Mutex + volumeGetCount int +} + +func newCinderAliasServer(t *testing.T, serviceType string) *cinderAliasServer { + t.Helper() + srv := &cinderAliasServer{} + mux := http.NewServeMux() + + mux.HandleFunc("/v3/auth/tokens", func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + w.WriteHeader(http.StatusMethodNotAllowed) + return + } + w.Header().Set("X-Subject-Token", "test-alias-token") + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusCreated) + _ = json.NewEncoder(w).Encode(map[string]any{ + "token": map[string]any{"user": map[string]any{"id": "test-user"}}, + }) + }) + + mux.HandleFunc("/v3/auth/catalog", func(w http.ResponseWriter, r *http.Request) { + selfURL := "http://" + r.Host + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{ + "catalog": []any{ + map[string]any{ + "type": serviceType, // "volume", "block-storage", … + "endpoints": []any{ + map[string]any{ + "interface": "public", + "region_id": "RegionOne", + "url": selfURL, + }, + }, + }, + }, + }) + }) + + mux.HandleFunc("/volumes/detail", func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + w.WriteHeader(http.StatusMethodNotAllowed) + return + } + srv.mu.Lock() + srv.volumeGetCount++ + srv.mu.Unlock() + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{"volumes": []any{}}) + }) + + srv.Server = httptest.NewServer(mux) + t.Cleanup(srv.Server.Close) + return srv +} + +func (srv *cinderAliasServer) authenticate(t *testing.T) *openstack.Session { + t.Helper() + cloudsYAML := fmt.Sprintf(` +clouds: + openstack: + auth_type: v3applicationcredential + auth: + auth_url: %s + application_credential_id: cred-id + application_credential_secret: cred-secret +`, srv.URL) + session, err := openstack.Authenticate(context.Background(), cloudsYAML, "openstack", "") + if err != nil { + t.Fatalf("authenticate: %v", err) + } + if !session.IsAuthenticated() { + t.Fatal("expected authenticated session") + } + return session +} + +// Scenario: catalog avec "block-storage" → cinderEndpoint résout le bon endpoint +func TestCinderEndpoint_FallsBackToBlockStorage(t *testing.T) { + srv := newCinderAliasServer(t, "block-storage") + session := srv.authenticate(t) + + err := session.DeleteVolumes(context.Background(), logr.Discard(), "test-cluster") + if err != nil { + t.Fatalf("DeleteVolumes with block-storage catalog: %v", err) + } + if srv.volumeGetCount == 0 { + t.Error("expected at least one GET /volumes/detail call") + } +} + +// Scenario: catalog avec "volume" uniquement → cinderEndpoint utilise l'alias legacy +func TestCinderEndpoint_FallsBackToVolumeAlias(t *testing.T) { + srv := newCinderAliasServer(t, "volume") + session := srv.authenticate(t) + + err := session.DeleteVolumes(context.Background(), logr.Discard(), "test-cluster") + if err != nil { + t.Fatalf("DeleteVolumes with volume alias catalog: %v", err) + } + if srv.volumeGetCount == 0 { + t.Error("expected at least one GET /volumes/detail call") + } +} From ca839bede5a914c48ddd122eb1bfee585ea1cd75 Mon Sep 17 00:00:00 2001 From: Mathieu Grzybek Date: Wed, 8 Jul 2026 13:33:59 +0200 Subject: [PATCH 14/27] feat(migration): add the roadmap Signed-off-by: Mathieu Grzybek --- ROADMAP.md | 982 ++++++++++++++++++++++++++++------------------------- 1 file changed, 518 insertions(+), 464 deletions(-) diff --git a/ROADMAP.md b/ROADMAP.md index 3f9a654..41bc943 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -1,649 +1,703 @@ -# Réécriture du projet en Go +# Go Rewrite -## Contexte +## Context -Le projet actuel est écrit en Python (asyncio + kopf + easykube + httpx). C'est un opérateur Kubernetes qui nettoie les ressources OpenStack laissées par l'OCCM et le CSI Cinder lors de la suppression de clusters Cluster API. +The current project is written in Python (asyncio + kopf + easykube + httpx). It is a Kubernetes operator that cleans up OpenStack resources left behind by OCCM and the Cinder CSI when Cluster API clusters are deleted. -La réécriture suit la méthode TDD : les tests Gherkin sont écrits en premier, puis les implémentations. +The rewrite follows TDD: Gherkin tests are written first, then the implementations. -Outil de scaffolding : **kubebuilder** (skill disponible). +Scaffolding tool: **kubebuilder**. --- -## Audit du code Python existant +## Audit of the Existing Python Code -### Modules principaux +### Main Modules -| Fichier | Rôle | +| File | Role | |---|---| -| `capi_janitor/openstack/openstack.py` | Client OpenStack : authentification, catalogue de services, ressources REST paginées | -| `capi_janitor/openstack/operator.py` | Logique opérateur : handlers kopf, filtres de ressources, purge OpenStack | - -### Fonctionnalités couvertes - -**Authentification OpenStack** -- Uniquement `v3applicationcredential` -- Gestion du token X-Auth-Token (refresh avec mutex asyncio) -- Support certificat CA personnalisé (cacert depuis le secret K8s) -- Catalogue de services filtré par interface (public/internal/admin) et région - -**Filtrage des ressources à nettoyer** -- Floating IPs : description `"Floating IP for Kubernetes external service … from cluster "` -- Load Balancers Octavia : nom `kube_service__*` -- Security Groups : description `"Security Group for Service LoadBalancer in cluster "` -- Volumes Cinder : métadonnée `cinder.csi.openstack.org/cluster == `, sauf propriété `janitor.capi.azimuth-cloud.com/keep == true` -- Snapshots Cinder : même métadonnée cluster - -**Politique de suppression** -- Volumes : configurable via env var `CAPI_JANITOR_DEFAULT_VOLUMES_POLICY` (défaut `delete`) et annotation `janitor.capi.stackhpc.com/volumes-policy` par cluster -- Application Credential : supprimé si annotation `janitor.capi.stackhpc.com/credential-policy: delete` sur le secret ET si c'est le dernier finalizer - -**Lifecycle Kubernetes** -- Finalizer `janitor.capi.stackhpc.com` sur `OpenStackCluster` -- Nom du cluster : label `cluster.x-k8s.io/cluster-name` en priorité, sinon `metadata.name` -- Retry via annotation aléatoire `janitor.capi.stackhpc.com/retry` (déclenche un nouvel événement) -- Backoff configurable `CAPI_JANITOR_RETRY_DEFAULT_DELAY` (défaut 60s) - -**Gestion des erreurs** -- HTTP 400/409 lors de la suppression : retry silencieux -- HTTP 404 lors de la récupération du catalogue : authentification considérée comme échouée (sans erreur fatale) -- HTTP 422 lors du patch des finalizers : `TemporaryError` kopf -- Erreur catalogue `volumev3` → fallback sur `block-storage` - -### Tests existants - -| Fichier | Ce qui est testé | +| `capi_janitor/openstack/openstack.py` | OpenStack client: authentication, service catalog, paginated REST resources | +| `capi_janitor/openstack/operator.py` | Operator logic: kopf handlers, resource filters, OpenStack purge | + +### Covered Features + +**OpenStack Authentication** +- Only `v3applicationcredential` +- X-Auth-Token management (refresh with asyncio mutex) +- Custom CA certificate support (cacert from K8s secret) +- Service catalog filtered by interface (public/internal/admin) and region + +**Resource Filtering** +- Floating IPs: description `"Floating IP for Kubernetes external service … from cluster "` +- Octavia Load Balancers: name `kube_service__*` +- Security Groups: description `"Security Group for Service LoadBalancer in cluster "` +- Cinder Volumes: metadata `cinder.csi.openstack.org/cluster == `, unless property `janitor.capi.azimuth-cloud.com/keep == true` +- Cinder Snapshots: same cluster metadata + +**Deletion Policy** +- Volumes: configurable via env var `CAPI_JANITOR_DEFAULT_VOLUMES_POLICY` (default `delete`) and annotation `janitor.capi.stackhpc.com/volumes-policy` per cluster +- Application Credential: deleted if annotation `janitor.capi.stackhpc.com/credential-policy: delete` on the secret AND it is the last finalizer + +**Kubernetes Lifecycle** +- Finalizer `janitor.capi.stackhpc.com` on `OpenStackCluster` +- Cluster name: label `cluster.x-k8s.io/cluster-name` takes priority, otherwise `metadata.name` +- Retry via random annotation `janitor.capi.stackhpc.com/retry` (triggers a new event) +- Configurable backoff `CAPI_JANITOR_RETRY_DEFAULT_DELAY` (default 60s) + +**Error Handling** +- HTTP 400/409 during deletion: silent retry +- HTTP 404 during catalog fetch: authentication considered failed (no fatal error) +- HTTP 422 during finalizer patch: kopf `TemporaryError` +- Catalog error `volumev3` → fallback to `block-storage` + +### Existing Tests + +| File | What is tested | |---|---| -| `test_openstack.py` | Authentification réussie, 404, absence d'interface, absence de région, multiples services | -| `test_operator.py` | Filtrage FIPs, LBs, SGs, volumes, snapshots ; `empty()` ; `try_delete()` ; handler d'événement (ajout finalizer, skip, purge) ; erreur d'auth dans purge | +| `test_openstack.py` | Successful auth, 404, missing interface, missing region, multiple services | +| `test_operator.py` | FIP/LB/SG/volume/snapshot filtering; `empty()`; `try_delete()`; event handler (add finalizer, skip, purge); auth error in purge | -**Lacune notable** : `test_purge_openstack_resources_success` est commenté (complexité du mock). +**Notable gap**: `test_purge_openstack_resources_success` is commented out (mock complexity). -### Chart Helm +### Helm Chart -- `ClusterRole` : namespaces (list/watch), events (create), secrets (get/delete), openstackclusters (list/get/watch/patch), CRDs (list/get/watch) -- Valeur `defaultVolumesPolicy: delete` -- Image : `ghcr.io/azimuth-cloud/cluster-api-janitor-openstack` +- `ClusterRole`: namespaces (list/watch), events (create), secrets (get/delete), openstackclusters (list/get/watch/patch), CRDs (list/get/watch) +- Value `defaultVolumesPolicy: delete` +- Image: `ghcr.io/azimuth-cloud/cluster-api-janitor-openstack` -### PRs en attente à intégrer +### Pending PRs to Integrate -| PR | Titre | Impact | +| PR | Title | Impact | |---|---|---| -| #261 | Fix leaving Azimuth cluster loadbalancers behind | Ajoute la détection des LBs Azimuth (`kube_service__` + LBs nommés différemment par Azimuth) | +| #261 | Fix leaving Azimuth cluster loadbalancers behind | Adds detection of Azimuth LBs (`kube_service__` + LBs named differently by Azimuth) | --- -## Feuille de route agile +## Agile Roadmap --- -### Epic 1 — Authentification OpenStack +### Epic 1 — OpenStack Authentication -#### US1.1 — Authentification via Application Credential v3 +#### US1.1 — Authentication via Application Credential v3 ```gherkin -Feature: Authentification OpenStack via Application Credential - In order to accéder aux APIs OpenStack - As an opérateur - I want to m'authentifier avec un Application Credential v3 - - Scenario: Authentification réussie - Given un clouds.yaml avec auth_type "v3applicationcredential" - And un application_credential_id et application_credential_secret valides - When l'opérateur initialise la connexion OpenStack - Then un token X-Auth-Token est obtenu depuis Keystone - And le catalogue de services est chargé - - Scenario: Refresh du token lors d'une expiration - Given un token X-Auth-Token expiré - When l'opérateur effectue un appel API - Then un nouveau token est demandé à Keystone - And l'appel original est rejoué avec le nouveau token - - Scenario: Authentification avec un type non supporté - Given un clouds.yaml avec auth_type "password" - When l'opérateur tente de créer un client Cloud - Then une erreur UnsupportedAuthenticationError est levée +Feature: OpenStack Authentication via Application Credential + In order to access OpenStack APIs + As an operator + I want to authenticate using a v3 Application Credential + + Scenario: Successful authentication + Given a clouds.yaml with auth_type "v3applicationcredential" + And a valid application_credential_id and application_credential_secret + When the operator initialises the OpenStack connection + Then an X-Auth-Token is obtained from Keystone + And the service catalog is loaded + + Scenario: Token refresh on expiry + Given an expired X-Auth-Token + When the operator makes an API call + Then a new token is requested from Keystone + And the original call is replayed with the new token + + Scenario: Authentication with unsupported type + Given a clouds.yaml with auth_type "password" + When the operator attempts to create a Cloud client + Then an UnsupportedAuthenticationError is raised ``` -#### US1.2 — Filtrage du catalogue de services par interface et région +#### US1.2 — Service Catalog Filtering by Interface and Region ```gherkin -Feature: Catalogue de services OpenStack - Scenario: Endpoint sélectionné selon l'interface configurée - Given un catalogue avec des endpoints "public" et "internal" - And l'interface configurée est "public" - When le catalogue est chargé - Then seuls les endpoints "public" sont retenus - - Scenario: Endpoint sélectionné selon la région configurée - Given un catalogue avec des endpoints pour "RegionOne" et "RegionTwo" - And la région configurée est "RegionOne" - When le catalogue est chargé - Then seuls les endpoints de "RegionOne" sont retenus - - Scenario: Aucune région configurée - Given un catalogue avec des endpoints dans plusieurs régions - And aucune région n'est configurée - When le catalogue est chargé - Then le premier endpoint correspondant à l'interface est retenu pour chaque service +Feature: OpenStack Service Catalog + Scenario: Endpoint selected by configured interface + Given a catalog with "public" and "internal" endpoints + And the configured interface is "public" + When the catalog is loaded + Then only "public" endpoints are retained + + Scenario: Endpoint selected by configured region + Given a catalog with endpoints for "RegionOne" and "RegionTwo" + And the configured region is "RegionOne" + When the catalog is loaded + Then only "RegionOne" endpoints are retained + + Scenario: No region configured + Given a catalog with endpoints in multiple regions + And no region is configured + When the catalog is loaded + Then the first endpoint matching the interface is retained for each service ``` -#### US1.3 — Gestion d'un credential révoqué ou invalide +#### US1.3 — Revoked or Invalid Credential Handling ```gherkin -Feature: Credential OpenStack invalide - Scenario: Application credential supprimé avant la purge - Given un cluster OpenStack en cours de suppression - And l'application credential a déjà été supprimé - When l'opérateur tente de s'authentifier - Then is_authenticated retourne false - And si include_appcred est true, un warning est émis et la purge s'arrête proprement - And si include_appcred est false, une AuthenticationError est levée - - Scenario: Catalogue retourne 404 - Given une URL Keystone valide mais le catalogue retourne 404 - When l'opérateur charge le catalogue - Then is_authenticated retourne false - And aucune erreur fatale n'est levée +Feature: Invalid OpenStack Credential + Scenario: Application credential deleted before purge + Given an OpenStack cluster being deleted + And the application credential has already been deleted + When the operator attempts to authenticate + Then is_authenticated returns false + And if include_appcred is true, a warning is emitted and the purge stops cleanly + And if include_appcred is false, an AuthenticationError is raised + + Scenario: Catalog returns 404 + Given a valid Keystone URL but the catalog returns 404 + When the operator loads the catalog + Then is_authenticated returns false + And no fatal error is raised ``` -#### US1.4 — Support des certificats CA personnalisés +#### US1.4 — Custom CA Certificate Support ```gherkin -Feature: Certificat CA personnalisé - Scenario: CA fourni dans le secret Kubernetes - Given un secret Kubernetes contenant une entrée "cacert" - When l'opérateur initialise le transport TLS - Then le CA est chargé dans le contexte SSL - And les appels HTTPS vers OpenStack utilisent ce CA pour la vérification - - Scenario: Pas de CA fourni - Given un secret Kubernetes sans entrée "cacert" - When l'opérateur initialise le transport TLS - Then le CA système est utilisé pour la vérification TLS +Feature: Custom CA Certificate + Scenario: CA provided in the Kubernetes secret + Given a Kubernetes secret containing a "cacert" entry + When the operator initialises the TLS transport + Then the CA is loaded into the SSL context + And HTTPS calls to OpenStack use this CA for verification + + Scenario: No CA provided + Given a Kubernetes secret without a "cacert" entry + When the operator initialises the TLS transport + Then the system CA is used for TLS verification ``` --- -### Epic 2 — Nettoyage des Floating IPs +### Epic 2 — Floating IP Cleanup -#### US2.1 — Identifier les Floating IPs d'un cluster +#### US2.1 — Identify Floating IPs of a Cluster ```gherkin -Feature: Identification des Floating IPs d'un cluster - Scenario: FIP appartenant au cluster - Given une liste de Floating IPs OpenStack - And une FIP avec la description "Floating IP for Kubernetes external service from cluster mycluster" - When les FIPs du cluster "mycluster" sont listées - Then cette FIP est incluse dans le résultat - - Scenario: FIP d'un autre cluster - Given une FIP avec la description "Floating IP for Kubernetes external service from cluster othercluster" - When les FIPs du cluster "mycluster" sont listées - Then cette FIP est exclue du résultat - - Scenario: FIP sans description Kubernetes - Given une FIP avec la description "Some other description" - When les FIPs du cluster "mycluster" sont listées - Then cette FIP est exclue du résultat +Feature: Identifying Floating IPs of a Cluster + Scenario: FIP belonging to the cluster + Given a list of OpenStack Floating IPs + And a FIP with description "Floating IP for Kubernetes external service from cluster mycluster" + When the FIPs of cluster "mycluster" are listed + Then this FIP is included in the result + + Scenario: FIP from another cluster + Given a FIP with description "Floating IP for Kubernetes external service from cluster othercluster" + When the FIPs of cluster "mycluster" are listed + Then this FIP is excluded from the result + + Scenario: FIP without a Kubernetes description + Given a FIP with description "Some other description" + When the FIPs of cluster "mycluster" are listed + Then this FIP is excluded from the result ``` -#### US2.2 — Supprimer les Floating IPs +#### US2.2 — Delete Floating IPs ```gherkin -Feature: Suppression des Floating IPs - Scenario: Suppression réussie - Given une FIP appartenant au cluster "mycluster" - When la purge des FIPs est déclenchée - Then la FIP est supprimée via l'API Neutron - And un log INFO est émis - - Scenario: Erreur HTTP 400 lors de la suppression - Given une suppression de FIP retourne HTTP 400 - When la purge tente de supprimer la FIP - Then un warning est émis - And la suppression continue pour les autres FIPs - And check_fips est true pour déclencher une vérification - - Scenario: Erreur HTTP 500 lors de la suppression - Given une suppression de FIP retourne HTTP 500 - When la purge tente de supprimer la FIP - Then une exception est propagée +Feature: Floating IP Deletion + Scenario: Successful deletion + Given a FIP belonging to cluster "mycluster" + When the FIP purge is triggered + Then the FIP is deleted via the Neutron API + And an INFO log is emitted + + Scenario: HTTP 400 error during deletion + Given a FIP deletion returns HTTP 400 + When the purge attempts to delete the FIP + Then a warning is emitted + And deletion continues for other FIPs + And check_fips is true to trigger a verification + + Scenario: HTTP 500 error during deletion + Given a FIP deletion returns HTTP 500 + When the purge attempts to delete the FIP + Then an exception is propagated ``` --- -### Epic 3 — Nettoyage des Load Balancers Octavia +### Epic 3 — Octavia Load Balancer Cleanup -#### US3.1 — Identifier les Load Balancers Kubernetes d'un cluster +#### US3.1 — Identify Kubernetes Load Balancers of a Cluster ```gherkin -Feature: Identification des Load Balancers Kubernetes - Scenario: LB appartenant au cluster - Given un LB avec le nom "kube_service_mycluster_api" - When les LBs du cluster "mycluster" sont listés - Then ce LB est inclus dans le résultat - - Scenario: LB d'un autre cluster - Given un LB avec le nom "kube_service_othercluster_api" - When les LBs du cluster "mycluster" sont listés - Then ce LB est exclu du résultat - - Scenario: LB sans préfixe kube_service - Given un LB avec le nom "fake_service_mycluster_api" - When les LBs du cluster "mycluster" sont listés - Then ce LB est exclu du résultat +Feature: Identifying Kubernetes Load Balancers + Scenario: LB belonging to the cluster + Given an LB with name "kube_service_mycluster_api" + When the LBs of cluster "mycluster" are listed + Then this LB is included in the result + + Scenario: LB from another cluster + Given an LB with name "kube_service_othercluster_api" + When the LBs of cluster "mycluster" are listed + Then this LB is excluded from the result + + Scenario: LB without kube_service prefix + Given an LB with name "fake_service_mycluster_api" + When the LBs of cluster "mycluster" are listed + Then this LB is excluded from the result ``` -#### US3.2 — Identifier les Load Balancers Azimuth (PR #261) +#### US3.2 — Identify Azimuth Load Balancers (PR #261) ```gherkin -Feature: Identification des Load Balancers Azimuth - Scenario: LB Azimuth appartenant au cluster - Given un LB Azimuth identifiable comme appartenant au cluster "mycluster" - When les LBs du cluster "mycluster" sont listés - Then ce LB Azimuth est inclus dans le résultat - - Scenario: Erreur HTTP lors du listing des LBs - Given l'API Octavia retourne une erreur HTTP lors du listing - When les LBs du cluster "mycluster" sont listés - Then un log ERROR est émis avec le code HTTP - And aucune exception n'est propagée - And un warning indique que des LBs pourraient rester +Feature: Identifying Azimuth Load Balancers + Scenario: Azimuth LB belonging to the cluster + Given an Azimuth LB identifiable as belonging to cluster "mycluster" + When the LBs of cluster "mycluster" are listed + Then this Azimuth LB is included in the result + + Scenario: HTTP error during LB listing + Given the Octavia API returns an HTTP error during listing + When the LBs of cluster "mycluster" are listed + Then an ERROR log is emitted with the HTTP code + And no exception is propagated + And a warning indicates that LBs may remain ``` -#### US3.3 — Supprimer les Load Balancers en cascade +#### US3.3 — Delete Load Balancers with Cascade ```gherkin -Feature: Suppression des Load Balancers en cascade - Scenario: Suppression réussie avec cascade - Given un LB appartenant au cluster "mycluster" - When la purge des LBs est déclenchée - Then le LB est supprimé avec le paramètre cascade=true - And les ressources Octavia associées (listeners, pools, membres) sont supprimées +Feature: Cascaded Load Balancer Deletion + Scenario: Successful deletion with cascade + Given an LB belonging to cluster "mycluster" + When the LB purge is triggered + Then the LB is deleted with the cascade=true parameter + And associated Octavia resources (listeners, pools, members) are deleted ``` --- -### Epic 4 — Nettoyage des Security Groups +### Epic 4 — Security Group Cleanup -#### US4.1 — Identifier les Security Groups d'un cluster +#### US4.1 — Identify Security Groups of a Cluster ```gherkin -Feature: Identification des Security Groups d'un cluster - Scenario: SG appartenant au cluster - Given un SG avec la description "Security Group for Service LoadBalancer in cluster mycluster" - When les SGs du cluster "mycluster" sont listés - Then ce SG est inclus dans le résultat - - Scenario: SG d'un autre cluster - Given un SG avec la description "Security Group for Service LoadBalancer in cluster othercluster" - When les SGs du cluster "mycluster" sont listés - Then ce SG est exclu du résultat +Feature: Identifying Security Groups of a Cluster + Scenario: SG belonging to the cluster + Given an SG with description "Security Group for Service LoadBalancer in cluster mycluster" + When the SGs of cluster "mycluster" are listed + Then this SG is included in the result + + Scenario: SG from another cluster + Given an SG with description "Security Group for Service LoadBalancer in cluster othercluster" + When the SGs of cluster "mycluster" are listed + Then this SG is excluded from the result ``` -#### US4.2 — Supprimer les Security Groups +#### US4.2 — Delete Security Groups ```gherkin -Feature: Suppression des Security Groups - Scenario: Suppression réussie - Given un SG appartenant au cluster "mycluster" - When la purge des SGs est déclenchée - Then le SG est supprimé via l'API Neutron - - Scenario: SG encore utilisé (HTTP 409) - Given une suppression de SG retourne HTTP 409 - When la purge tente de supprimer le SG - Then un warning est émis - And check_secgroups est true pour une vérification ultérieure +Feature: Security Group Deletion + Scenario: Successful deletion + Given an SG belonging to cluster "mycluster" + When the SG purge is triggered + Then the SG is deleted via the Neutron API + + Scenario: SG still in use (HTTP 409) + Given an SG deletion returns HTTP 409 + When the purge attempts to delete the SG + Then a warning is emitted + And check_secgroups is true for a later verification ``` --- -### Epic 5 — Gestion des Volumes Cinder +### Epic 5 — Cinder Volume Management -#### US5.1 — Identifier les volumes d'un cluster +#### US5.1 — Identify Volumes of a Cluster ```gherkin -Feature: Identification des volumes Cinder d'un cluster - Scenario: Volume appartenant au cluster sans marquage keep - Given un volume avec la métadonnée "cinder.csi.openstack.org/cluster" = "mycluster" - And la propriété "janitor.capi.azimuth-cloud.com/keep" est absente ou != "true" - When les volumes du cluster "mycluster" sont listés - Then ce volume est inclus dans le résultat - - Scenario: Volume marqué keep par l'utilisateur - Given un volume avec la métadonnée "cinder.csi.openstack.org/cluster" = "mycluster" - And la propriété "janitor.capi.azimuth-cloud.com/keep" = "true" - When les volumes du cluster "mycluster" sont listés - Then ce volume est exclu du résultat - - Scenario: Volume d'un autre cluster - Given un volume avec la métadonnée "cinder.csi.openstack.org/cluster" = "othercluster" - When les volumes du cluster "mycluster" sont listés - Then ce volume est exclu du résultat - - Scenario: Volume sans métadonnée CSI - Given un volume sans métadonnée "cinder.csi.openstack.org/cluster" - When les volumes du cluster "mycluster" sont listés - Then ce volume est exclu du résultat +Feature: Identifying Cinder Volumes of a Cluster + Scenario: Volume belonging to the cluster without keep flag + Given a volume with metadata "cinder.csi.openstack.org/cluster" = "mycluster" + And the property "janitor.capi.azimuth-cloud.com/keep" is absent or != "true" + When the volumes of cluster "mycluster" are listed + Then this volume is included in the result + + Scenario: Volume flagged keep by the user + Given a volume with metadata "cinder.csi.openstack.org/cluster" = "mycluster" + And the property "janitor.capi.azimuth-cloud.com/keep" = "true" + When the volumes of cluster "mycluster" are listed + Then this volume is excluded from the result + + Scenario: Volume from another cluster + Given a volume with metadata "cinder.csi.openstack.org/cluster" = "othercluster" + When the volumes of cluster "mycluster" are listed + Then this volume is excluded from the result + + Scenario: Volume without CSI metadata + Given a volume without metadata "cinder.csi.openstack.org/cluster" + When the volumes of cluster "mycluster" are listed + Then this volume is excluded from the result ``` -#### US5.2 — Politique de suppression des volumes +#### US5.2 — Volume Deletion Policy ```gherkin -Feature: Politique de suppression des volumes - Scenario: Politique globale "delete" (défaut) - Given la variable d'environnement CAPI_JANITOR_DEFAULT_VOLUMES_POLICY non définie - When un cluster est supprimé sans annotation de volumes - Then les volumes du cluster sont supprimés +Feature: Volume Deletion Policy + Scenario: Global policy "delete" (default) + Given the environment variable CAPI_JANITOR_DEFAULT_VOLUMES_POLICY is not set + When a cluster is deleted without a volumes annotation + Then the cluster's volumes are deleted - Scenario: Politique globale "keep" + Scenario: Global policy "keep" Given CAPI_JANITOR_DEFAULT_VOLUMES_POLICY = "keep" - When un cluster est supprimé sans annotation de volumes - Then les volumes du cluster sont conservés + When a cluster is deleted without a volumes annotation + Then the cluster's volumes are kept - Scenario: Annotation "delete" sur le cluster (override keep global) + Scenario: Annotation "delete" on the cluster (overrides global keep) Given CAPI_JANITOR_DEFAULT_VOLUMES_POLICY = "keep" - And l'annotation "janitor.capi.stackhpc.com/volumes-policy" = "delete" sur l'OpenStackCluster - When le cluster est supprimé - Then les volumes du cluster sont supprimés + And the annotation "janitor.capi.stackhpc.com/volumes-policy" = "delete" on the OpenStackCluster + When the cluster is deleted + Then the cluster's volumes are deleted - Scenario: Annotation "keep" sur le cluster (override delete global) + Scenario: Annotation "keep" on the cluster (overrides global delete) Given CAPI_JANITOR_DEFAULT_VOLUMES_POLICY = "delete" - And l'annotation "janitor.capi.stackhpc.com/volumes-policy" = "keep" sur l'OpenStackCluster - When le cluster est supprimé - Then les volumes du cluster sont conservés + And the annotation "janitor.capi.stackhpc.com/volumes-policy" = "keep" on the OpenStackCluster + When the cluster is deleted + Then the cluster's volumes are kept ``` --- -### Epic 6 — Gestion des Snapshots Cinder +### Epic 6 — Cinder Snapshot Management -#### US6.1 — Identifier et supprimer les snapshots d'un cluster +#### US6.1 — Identify and Delete Snapshots of a Cluster ```gherkin -Feature: Snapshots Cinder d'un cluster - Scenario: Snapshot appartenant au cluster - Given un snapshot avec la métadonnée "cinder.csi.openstack.org/cluster" = "mycluster" - When les snapshots du cluster "mycluster" sont listés - Then ce snapshot est inclus dans le résultat - - Scenario: Snapshot d'un autre cluster - Given un snapshot avec la métadonnée "cinder.csi.openstack.org/cluster" = "othercluster" - When les snapshots du cluster "mycluster" sont listés - Then ce snapshot est exclu du résultat - - Scenario: Snapshots supprimés avant les volumes - Given des snapshots et des volumes appartenant au cluster "mycluster" - When la purge est déclenchée avec include_volumes = true - Then les snapshots sont supprimés en premier - And les volumes sont supprimés ensuite +Feature: Cinder Snapshots of a Cluster + Scenario: Snapshot belonging to the cluster + Given a snapshot with metadata "cinder.csi.openstack.org/cluster" = "mycluster" + When the snapshots of cluster "mycluster" are listed + Then this snapshot is included in the result + + Scenario: Snapshot from another cluster + Given a snapshot with metadata "cinder.csi.openstack.org/cluster" = "othercluster" + When the snapshots of cluster "mycluster" are listed + Then this snapshot is excluded from the result + + Scenario: Snapshots deleted before volumes + Given snapshots and volumes belonging to cluster "mycluster" + When the purge is triggered with include_volumes = true + Then snapshots are deleted first + And volumes are deleted afterwards ``` --- -### Epic 7 — Gestion des Application Credentials +### Epic 7 — Application Credential Management -#### US7.1 — Supprimer l'Application Credential OpenStack +#### US7.1 — Delete the OpenStack Application Credential ```gherkin -Feature: Suppression de l'Application Credential - Scenario: Suppression autorisée (dernier finalizer) - Given l'annotation "janitor.capi.stackhpc.com/credential-policy" = "delete" sur le secret - And le finalizer de l'opérateur est le seul finalizer présent - When la purge des ressources OpenStack est terminée - Then l'Application Credential est supprimé via l'API Identity - And le secret Kubernetes contenant clouds.yaml est supprimé - - Scenario: Autres finalizers encore présents - Given l'annotation "credential-policy" = "delete" sur le secret - And d'autres finalizers sont encore présents sur l'OpenStackCluster - When la purge est terminée - Then l'Application Credential n'est pas supprimé - And une FinalizerStillPresentError est levée pour déclencher un retry - - Scenario: Application Credential non supprimable (403) - Given l'Application Credential est restreint (pas d'unrestricted) - When la suppression de l'appcred est tentée - Then un warning est émis - And la suppression du secret Kubernetes procède quand même +Feature: Application Credential Deletion + Scenario: Deletion authorised (last finalizer) + Given the annotation "janitor.capi.stackhpc.com/credential-policy" = "delete" on the secret + And the operator's finalizer is the only finalizer present + When the purge of OpenStack resources is complete + Then the Application Credential is deleted via the Identity API + And the Kubernetes secret containing clouds.yaml is deleted + + Scenario: Other finalizers still present + Given the annotation "credential-policy" = "delete" on the secret + And other finalizers are still present on the OpenStackCluster + When the purge is complete + Then the Application Credential is not deleted + And a FinalizerStillPresentError is raised to trigger a retry + + Scenario: Application Credential cannot be deleted (403) + Given the Application Credential is restricted (no unrestricted flag) + When the appcred deletion is attempted + Then a warning is emitted + And the Kubernetes secret deletion proceeds anyway ``` --- -### Epic 8 — Lifecycle Kubernetes (pattern Finalizer) +### Epic 8 — Kubernetes Lifecycle (Finalizer Pattern) -#### US8.1 — Ajouter un finalizer à la création +#### US8.1 — Add a Finalizer on Creation ```gherkin -Feature: Ajout du finalizer janitor sur OpenStackCluster - Scenario: Cluster sans deletionTimestamp et sans finalizer janitor - Given un OpenStackCluster sans deletionTimestamp - And sans finalizer "janitor.capi.stackhpc.com" - When un événement est reçu pour ce cluster - Then le finalizer "janitor.capi.stackhpc.com" est ajouté via patch - And un log INFO confirme l'ajout - - Scenario: Cluster avec finalizer déjà présent - Given un OpenStackCluster sans deletionTimestamp - And avec le finalizer "janitor.capi.stackhpc.com" déjà présent - When un événement est reçu - Then aucun patch n'est effectué +Feature: Adding the Janitor Finalizer to OpenStackCluster + Scenario: Cluster without deletionTimestamp and without janitor finalizer + Given an OpenStackCluster without deletionTimestamp + And without finalizer "janitor.capi.stackhpc.com" + When an event is received for this cluster + Then the finalizer "janitor.capi.stackhpc.com" is added via patch + And an INFO log confirms the addition + + Scenario: Cluster with finalizer already present + Given an OpenStackCluster without deletionTimestamp + And with the finalizer "janitor.capi.stackhpc.com" already present + When an event is received + Then no patch is made ``` -#### US8.2 — Nom du cluster depuis le label ou metadata.name +#### US8.2 — Cluster Name from Label or metadata.name ```gherkin -Feature: Résolution du nom du cluster - Scenario: Label cluster.x-k8s.io/cluster-name présent - Given un OpenStackCluster avec le label "cluster.x-k8s.io/cluster-name" = "myapp" +Feature: Cluster Name Resolution + Scenario: Label cluster.x-k8s.io/cluster-name present + Given an OpenStackCluster with label "cluster.x-k8s.io/cluster-name" = "myapp" And metadata.name = "myapp-openstack" - When l'opérateur résout le nom du cluster pour le nettoyage - Then le nom "myapp" est utilisé + When the operator resolves the cluster name for cleanup + Then the name "myapp" is used Scenario: Label absent - Given un OpenStackCluster sans label "cluster.x-k8s.io/cluster-name" + Given an OpenStackCluster without label "cluster.x-k8s.io/cluster-name" And metadata.name = "mycluster" - When l'opérateur résout le nom du cluster - Then le nom "mycluster" est utilisé + When the operator resolves the cluster name + Then the name "mycluster" is used ``` -#### US8.3 — Supprimer le finalizer après nettoyage réussi +#### US8.3 — Remove the Finalizer after Successful Cleanup ```gherkin -Feature: Suppression du finalizer après purge - Scenario: Purge réussie - Given un OpenStackCluster en cours de suppression - And toutes les ressources OpenStack ont été supprimées - When la purge est terminée sans erreur - Then le finalizer "janitor.capi.stackhpc.com" est retiré via patch - And un log INFO confirme la suppression du finalizer - - Scenario: Finalizer absent au moment de la suppression - Given un OpenStackCluster avec deletionTimestamp - And sans finalizer "janitor.capi.stackhpc.com" - When un événement est reçu - Then aucune purge n'est déclenchée - And un log INFO indique que le finalizer est absent +Feature: Finalizer Removal after Purge + Scenario: Successful purge + Given an OpenStackCluster being deleted + And all OpenStack resources have been deleted + When the purge completes without error + Then the finalizer "janitor.capi.stackhpc.com" is removed via patch + And an INFO log confirms the finalizer removal + + Scenario: Finalizer absent at removal time + Given an OpenStackCluster with deletionTimestamp + And without the finalizer "janitor.capi.stackhpc.com" + When an event is received + Then no purge is triggered + And an INFO log indicates the finalizer is absent ``` -#### US8.4 — Mécanisme de retry via annotation +#### US8.4 — Retry Mechanism via Annotation ```gherkin -Feature: Retry via annotation aléatoire - Scenario: Erreur temporaire lors de la purge - Given une purge qui échoue avec une ResourcesStillPresentError - When l'opérateur gère l'erreur - Then après un délai de backoff (5s pour ResourcesStillPresent) - And une annotation aléatoire "janitor.capi.stackhpc.com/retry" est posée sur l'OpenStackCluster - And un nouvel événement est déclenché pour rejouer la purge - - Scenario: Erreur inconnue lors de la purge - Given une purge qui échoue avec une exception non classifiée - When l'opérateur gère l'erreur - Then le délai est CAPI_JANITOR_RETRY_DEFAULT_DELAY (défaut 60s) - And l'exception est loguée avec stack trace - - Scenario: Ressource supprimée entre l'erreur et le retry - Given l'OpenStackCluster est supprimé pendant le backoff - When l'opérateur tente d'annoter la ressource - Then l'ApiError 404 est ignorée +Feature: Retry via Random Annotation + Scenario: Transient error during purge + Given a purge that fails with a ResourcesStillPresentError + When the operator handles the error + Then after a backoff delay (5s for ResourcesStillPresent) + And a random annotation "janitor.capi.stackhpc.com/retry" is set on the OpenStackCluster + And a new event is triggered to replay the purge + + Scenario: Unknown error during purge + Given a purge that fails with an unclassified exception + When the operator handles the error + Then the delay is CAPI_JANITOR_RETRY_DEFAULT_DELAY (default 60s) + And the exception is logged with a stack trace + + Scenario: Resource deleted between the error and the retry + Given the OpenStackCluster is deleted during backoff + When the operator attempts to annotate the resource + Then the 404 ApiError is ignored ``` --- -### Epic 9 — Configuration de l'opérateur +### Epic 9 — Operator Configuration -#### US9.1 — Configuration via variables d'environnement +#### US9.1 — Configuration via Environment Variables ```gherkin -Feature: Configuration via variables d'environnement - Scenario: Politique volumes par défaut configurée +Feature: Configuration via Environment Variables + Scenario: Default volumes policy configured Given CAPI_JANITOR_DEFAULT_VOLUMES_POLICY = "keep" - When l'opérateur démarre - Then la politique par défaut pour tous les clusters est "keep" + When the operator starts + Then the default policy for all clusters is "keep" - Scenario: Délai de retry configurable + Scenario: Configurable retry delay Given CAPI_JANITOR_RETRY_DEFAULT_DELAY = "120" - When une erreur non classifiée se produit - Then le délai de retry est 120 secondes + When an unclassified error occurs + Then the retry delay is 120 seconds ``` --- -### Epic 10 — Packaging et déploiement +### Epic 10 — Packaging and Deployment -#### US10.1 — Image Docker +#### US10.1 — Secure Image (Go Dockerfile) ```gherkin -Feature: Image Docker de l'opérateur - Scenario: Build et push de l'image - Given le code source de l'opérateur Go - When le workflow GitHub Actions build-push-artifacts est déclenché - Then une image est publiée sur ghcr.io/azimuth-cloud/cluster-api-janitor-openstack - And l'image est taguée avec la version du chart - - Scenario: Sécurité de l'image - Given l'image Docker - Then le processus tourne en tant que non-root - And le système de fichiers racine est en lecture seule - And toutes les capabilities Linux sont droppées +Feature: Secure OCI Image for the Go Operator + Scenario: Multi-stage Go build + Given the Go operator source code + When the Dockerfile is built + Then the builder uses golang:1.26 + And the runtime uses gcr.io/distroless/static:nonroot (UID 65532) + + Scenario: Image security + Given the built image + Then the process runs as non-root (UID 65532) + And the root filesystem is read-only + And all Linux capabilities are dropped ``` -#### US10.2 — Helm chart +#### US10.2 — Helm Chart ```gherkin -Feature: Déploiement via Helm chart - Scenario: Installation avec les valeurs par défaut - Given le Helm chart cluster-api-janitor-openstack - When helm install est exécuté - Then un Deployment, ServiceAccount, ClusterRole et ClusterRoleBinding sont créés - And la politique volumes par défaut est "delete" - - Scenario: Override de la politique volumes - Given helm install avec --set defaultVolumesPolicy=keep - When le chart est déployé - Then la variable CAPI_JANITOR_DEFAULT_VOLUMES_POLICY = "keep" est injectée dans le pod +Feature: Deployment via Helm Chart + Scenario: Installation with default values + Given the cluster-api-janitor-openstack Helm chart + When helm install is executed + Then a Deployment, ServiceAccount, ClusterRole, and ClusterRoleBinding are created + And the default volumes policy is "delete" + And the default retry delay is 60 seconds + + Scenario: Override volumes policy + Given helm install with --set defaultVolumesPolicy=keep + When the chart is deployed + Then the variable CAPI_JANITOR_DEFAULT_VOLUMES_POLICY = "keep" is injected into the pod + + Scenario: Health probes active + Given the deployed Deployment + Then a livenessProbe on /healthz:8081 is configured + And a readinessProbe on /readyz:8081 is configured + + Scenario: Complete RBAC + Given the deployed ClusterRole + Then the "update" verb is present on openstackclusters + (required for r.Update during finalizer management) +``` + +#### US10.3 — OCI Build via Nix (without Flake) and SBOM + +```gherkin +Feature: Reproducible OCI Build via Nix and SBOM Generation + Scenario: Build the amd64 image with Nix + Given the nix/default.nix file + When nix-build nix -A image is executed + Then an amd64 OCI image is produced (dockerTools.buildLayeredImage) + And the image runs as User 65532:65532 + + Scenario: Build the arm64 image by cross-compilation + Given the nix/default.nix file + When nix-build nix -A image-arm64 is executed + Then an arm64 OCI image is produced via pkgsCross.aarch64-multiplatform + And both images are combined into a multi-arch manifest via skopeo + docker manifest + + Scenario: CycloneDX SBOM generation + Given the compiled Go binary + When nix-build nix -A sbom is executed + Then an sbom.cdx.json file in CycloneDX format is produced + And it lists all Go modules (extracted from the buildinfo embedded in the binary) + And it is uploaded as an artefact of the GitHub Actions workflow ``` --- -### Epic 11 — Observabilité (nouvelle fonctionnalité) +### Epic 11 — Observability -#### US11.1 — Métriques Prometheus +#### US11.1 — Prometheus Metrics ```gherkin -Feature: Métriques Prometheus - Scenario: Comptage des purges réussies - Given une purge de cluster réussie - When les métriques sont exposées sur /metrics - Then le compteur "janitor_purge_total{status=success}" est incrémenté - - Scenario: Comptage des ressources supprimées par type - Given une purge ayant supprimé 3 FIPs, 2 LBs et 1 SG - When les métriques sont exposées - Then les jauges correspondantes reflètent les suppressions - - Scenario: Durée de purge - Given une purge terminée - When les métriques sont exposées - Then un histogramme "janitor_purge_duration_seconds" contient la durée de la purge +Feature: Prometheus Metrics + Scenario: Successful purge → success counter incremented + Given a cluster being deleted + When the OpenStack purge succeeds + Then capi_janitor_cleanups_total{result="success"} is incremented by 1 + + Scenario: Failed purge → failure counter incremented + Given a cluster being deleted + When the OpenStack purge fails + Then capi_janitor_cleanups_total{result="failure"} is incremented by 1 ``` -#### US11.2 — Status conditions sur OpenStackCluster +> Implemented: `CounterVec` exposed via `ctrlmetrics.Registry` (port 8080/metrics). +> `Metrics *Metrics` field is injectable on the reconciler (DI for tests). + +#### US11.2 — Kubernetes Events ```gherkin -Feature: Status conditions sur OpenStackCluster - Scenario: Purge en cours - Given une purge démarrée pour le cluster "mycluster" - When la purge est en cours - Then une condition "JanitorCleanupComplete" avec status "False" et reason "CleanupInProgress" est posée - - Scenario: Purge terminée avec succès - Given une purge réussie - When le finalizer est retiré - Then la condition "JanitorCleanupComplete" passe à status "True" - - Scenario: Purge en erreur - Given une purge échouant avec une erreur - When l'erreur est gérée - Then la condition "JanitorCleanupComplete" a status "False" et reason "CleanupFailed" avec un message d'erreur +Feature: Kubernetes Events on OpenStackCluster + Scenario: Successful purge → Normal "CleanupSucceeded" event + Given a cluster being deleted + When the OpenStack purge succeeds + Then a Normal event with reason "CleanupSucceeded" is emitted on the OpenStackCluster + + Scenario: Failed purge → Warning "CleanupFailed" event + Given a cluster being deleted + When the OpenStack purge fails + Then a Warning event with reason "CleanupFailed" and the error message is emitted ``` +> Implemented: `record.EventRecorder` injectable on the reconciler; `SetupWithManager` +> auto-initialises via `mgr.GetEventRecorderFor("capi-janitor")` when nil. + --- -### Epic 12 — Robustesse et extensibilité (nouvelles fonctionnalités) +### Epic 12 — Robustness -#### US12.1 — Timeout configurable pour les appels OpenStack +#### US12.1 — HTTP Client Timeout ```gherkin -Feature: Timeout HTTP configurable - Scenario: Appel OpenStack qui dépasse le timeout - Given CAPI_JANITOR_OPENSTACK_TIMEOUT = "30" - When un appel API OpenStack dépasse 30 secondes - Then le timeout est déclenché - And une erreur temporaire est loguée pour retry +Feature: HTTP Timeout on the OpenStack Client + Scenario: Context cancelled before the call + Given an already-cancelled context + When Authenticate is called + Then an error is returned immediately + + Scenario: Safety net on the http.Client + Given no context with a deadline provided by the caller + Then the http.Client has a Timeout of 30 seconds + (prevents calls from blocking indefinitely when OpenStack is unreachable) ``` -#### US12.2 — Support des services Cinder avec alias +#### US12.2 — Cinder Service Legacy Aliases ```gherkin -Feature: Détection du service Cinder avec alias - Scenario: Catalogue avec "volumev3" - Given un catalogue OpenStack avec le service type "volumev3" - When l'opérateur cherche le client Cinder - Then le client "volumev3" est utilisé - - Scenario: Catalogue avec "block-storage" uniquement - Given un catalogue OpenStack sans "volumev3" mais avec "block-storage" - When l'opérateur cherche le client Cinder - Then le client "block-storage" est utilisé - - Scenario: Catalogue sans service Cinder - Given un catalogue sans "volumev3" ni "block-storage" - When l'opérateur cherche le client Cinder - Then une CatalogError est levée avec le message approprié +Feature: Cinder Service Detection with Aliases + Scenario: Catalog with "volumev3" (standard >= Stein) + Given an OpenStack catalog with service type "volumev3" + When the operator looks up the Cinder client + Then the "volumev3" client is used + + Scenario: Catalog with "block-storage" only + Given an OpenStack catalog without "volumev3" but with "block-storage" + When the operator looks up the Cinder client + Then the "block-storage" client is used + + Scenario: Catalog with "volume" only (legacy alias < Stein) + Given an OpenStack catalog without "volumev3" or "block-storage" but with "volume" + When the operator looks up the Cinder client + Then the "volume" client is used + + Scenario: Catalog without a Cinder service + Given a catalog without "volumev3", "block-storage", or "volume" + When the operator looks up the Cinder client + Then a CatalogError is raised with the appropriate message ``` --- ## Actions -1. [x] Réaliser un audit du code -2. [ ] Écrire la feuille de route agile (ce document) -3. [ ] Scaffolding du projet Go avec kubebuilder (`/kubebuilder`) -4. [ ] Écrire les tests Go (TDD) pour chaque user story -5. [ ] Implémenter les fonctionnalités en Go -6. [ ] Migrer le Helm chart pour l'image Go -7. [ ] Implémenter les epics 11 (observabilité) et 12 (robustesse) +1. [x] Audit the existing code +2. [x] Write the agile roadmap (this document) +3. [x] Scaffold the Go project with kubebuilder +4. [x] Write Go tests (TDD) for each user story +5. [x] Implement the features in Go (108 tests) +6. [x] Migrate the Helm chart for the Go image +7. [x] Implement epics 11 (observability) and 12 (robustness) +8. [x] OCI build via Nix (without Flake) + CycloneDX SBOM (US10.3 — outside initial plan) + +## Final Result + +| Layer | Key Files | +|---|---| +| OpenStack client | `internal/openstack/cloud.go`, `resources.go`, `purge.go` | +| Controller | `internal/controller/openstackcluster_controller.go`, `metrics.go` | +| Config | `internal/controller/config.go` (env vars) | +| Tests | 108 tests (4 packages) | +| Packaging | `Dockerfile`, `nix/default.nix`, `nix/nixpkgs.nix` | +| Helm | `chart/` — Deployment, ClusterRole, RBAC, health probes | +| CI | `.github/workflows/build-push-artifacts.yaml` (Nix + skopeo + SBOM) | -## Ordre d'implémentation suggéré +## Implementation Order ``` Epic 1 (Auth) → Epic 2 (FIPs) → Epic 3 (LBs + PR #261) → Epic 4 (SGs) → Epic 5 (Volumes) → Epic 6 (Snapshots) → Epic 7 (AppCreds) → Epic 8 (Lifecycle K8s) → Epic 9 (Config) -→ Epic 10 (Packaging) → Epic 11 (Observabilité) → Epic 12 (Robustesse) +→ Epic 10 (Packaging + Nix/SBOM) → Epic 11 (Observability) +→ Epic 12 (Robustness) ``` From 8adac16979dd1c4ecbb0a5c38571583f5a692ddf Mon Sep 17 00:00:00 2001 From: Mathieu Grzybek Date: Wed, 8 Jul 2026 13:36:14 +0200 Subject: [PATCH 15/27] feat(doc): translate everything in English Signed-off-by: Mathieu Grzybek --- internal/controller/config_test.go | 20 +-- internal/controller/observability_test.go | 12 +- .../openstackcluster_controller_test.go | 24 +-- internal/openstack/cloud_test.go | 116 +++++++------- internal/openstack/resources_test.go | 144 +++++++++--------- internal/openstack/robustness_test.go | 6 +- 6 files changed, 161 insertions(+), 161 deletions(-) diff --git a/internal/controller/config_test.go b/internal/controller/config_test.go index ab58434..b859edc 100644 --- a/internal/controller/config_test.go +++ b/internal/controller/config_test.go @@ -12,9 +12,9 @@ import ( "github.com/azimuth-cloud/cluster-api-janitor-openstack/internal/openstack" ) -// ── US9.1: Configuration via variables d'environnement ──────────────────────── +// ── US9.1: Configuration via environment variables ──────────────────────── -// Scenario: CAPI_JANITOR_DEFAULT_VOLUMES_POLICY non définie → "delete" +// Scenario: CAPI_JANITOR_DEFAULT_VOLUMES_POLICY not set → "delete" func TestDefaultVolumesFromEnv_DefaultsToDelete(t *testing.T) { t.Setenv("CAPI_JANITOR_DEFAULT_VOLUMES_POLICY", "") if got := controller.DefaultVolumesFromEnv(); got != controller.PolicyDelete { @@ -30,7 +30,7 @@ func TestDefaultVolumesFromEnv_ReadsKeep(t *testing.T) { } } -// Scenario: CAPI_JANITOR_RETRY_DEFAULT_DELAY non définie → 60s (défaut) +// Scenario: CAPI_JANITOR_RETRY_DEFAULT_DELAY not set → 60s (default) func TestRetryDelayFromEnv_DefaultsTo60(t *testing.T) { t.Setenv("CAPI_JANITOR_RETRY_DEFAULT_DELAY", "") if got := controller.RetryDelayFromEnv(); got != 60 { @@ -54,7 +54,7 @@ func TestRetryDelayFromEnv_InvalidValue_FallsBackToDefault(t *testing.T) { } } -// ── Politique volumes via le reconciler ─────────────────────────────────────── +// ── Volume policy via the reconciler ─────────────────────────────────────── func withVolumePolicy(policy string) func(*infrav1.OpenStackCluster) { return func(c *infrav1.OpenStackCluster) { @@ -84,7 +84,7 @@ func reconcileForVolumesCapture(t *testing.T, defaultPolicy string, clusterOpts return captured } -// Scenario: Politique globale "delete" → volumes inclus dans la purge +// Scenario: Global policy "delete" → volumes included in the purge func TestReconcile_VolumesPolicy_IncludesVolumes_WhenDelete(t *testing.T) { opts := reconcileForVolumesCapture(t, controller.PolicyDelete) if !opts.IncludeVolumes { @@ -92,7 +92,7 @@ func TestReconcile_VolumesPolicy_IncludesVolumes_WhenDelete(t *testing.T) { } } -// Scenario: Politique globale "keep" → volumes exclus de la purge +// Scenario: Global policy "keep" → volumes excluded from the purge func TestReconcile_VolumesPolicy_ExcludesVolumes_WhenKeep(t *testing.T) { opts := reconcileForVolumesCapture(t, "keep") if opts.IncludeVolumes { @@ -100,7 +100,7 @@ func TestReconcile_VolumesPolicy_ExcludesVolumes_WhenKeep(t *testing.T) { } } -// Scenario: Annotation "delete" sur le cluster (override keep global) +// Scenario: Annotation "delete" on the cluster (overrides global keep) func TestReconcile_VolumesPolicy_AnnotationDeleteOverridesKeepGlobal(t *testing.T) { opts := reconcileForVolumesCapture(t, "keep", withVolumePolicy(controller.PolicyDelete)) if !opts.IncludeVolumes { @@ -108,7 +108,7 @@ func TestReconcile_VolumesPolicy_AnnotationDeleteOverridesKeepGlobal(t *testing. } } -// Scenario: Annotation "keep" sur le cluster (override delete global) +// Scenario: Annotation "keep" on the cluster (overrides global delete) func TestReconcile_VolumesPolicy_AnnotationKeepOverridesDeleteGlobal(t *testing.T) { opts := reconcileForVolumesCapture(t, controller.PolicyDelete, withVolumePolicy("keep")) if opts.IncludeVolumes { @@ -116,9 +116,9 @@ func TestReconcile_VolumesPolicy_AnnotationKeepOverridesDeleteGlobal(t *testing. } } -// ── Délai de retry configurable ─────────────────────────────────────────────── +// ── Configurable retry delay ─────────────────────────────────────────────── -// Scenario: RetryDefaultDelay = 120 → sleep appelé avec 120 secondes +// Scenario: RetryDefaultDelay = 120 → sleep called with 120 seconds func TestReconcile_RetryDelay_UsesConfiguredDelay(t *testing.T) { cluster := newCluster("mycluster", "default", withFinalizer, diff --git a/internal/controller/observability_test.go b/internal/controller/observability_test.go index 54f0dea..66b90a6 100644 --- a/internal/controller/observability_test.go +++ b/internal/controller/observability_test.go @@ -27,9 +27,9 @@ func newObsReconciler( return r, rec } -// ── US11.1 : Métriques Prometheus ──────────────────────────────────────────── +// ── US11.1: Prometheus Metrics ──────────────────────────────────────────── -// Scenario: nettoyage réussi → capi_janitor_cleanups_total{result="success"} += 1 +// Scenario: successful cleanup → capi_janitor_cleanups_total{result="success"} += 1 func TestMetrics_IncrementsSuccess_OnCleanup(t *testing.T) { cluster := newCluster("c", "default", withFinalizer, withDeletionTimestamp) secret := newSecret("cloud-credentials", "default") @@ -49,7 +49,7 @@ func TestMetrics_IncrementsSuccess_OnCleanup(t *testing.T) { } } -// Scenario: purge échouée → capi_janitor_cleanups_total{result="failure"} += 1 +// Scenario: failed purge → capi_janitor_cleanups_total{result="failure"} += 1 func TestMetrics_IncrementsFailure_OnPurgeError(t *testing.T) { cluster := newCluster("c", "default", withFinalizer, withDeletionTimestamp) secret := newSecret("cloud-credentials", "default") @@ -69,9 +69,9 @@ func TestMetrics_IncrementsFailure_OnPurgeError(t *testing.T) { } } -// ── US11.2 : Événements Kubernetes ─────────────────────────────────────────── +// ── US11.2: Kubernetes Events ─────────────────────────────────────────────── -// Scenario: nettoyage réussi → événement Normal "CleanupSucceeded" +// Scenario: successful cleanup → Normal "CleanupSucceeded" event func TestEvents_EmitsNormal_OnCleanupSuccess(t *testing.T) { cluster := newCluster("c", "default", withFinalizer, withDeletionTimestamp) secret := newSecret("cloud-credentials", "default") @@ -96,7 +96,7 @@ func TestEvents_EmitsNormal_OnCleanupSuccess(t *testing.T) { } } -// Scenario: purge échouée → événement Warning "CleanupFailed" +// Scenario: failed purge → Warning "CleanupFailed" event func TestEvents_EmitsWarning_OnPurgeFailure(t *testing.T) { cluster := newCluster("c", "default", withFinalizer, withDeletionTimestamp) secret := newSecret("cloud-credentials", "default") diff --git a/internal/controller/openstackcluster_controller_test.go b/internal/controller/openstackcluster_controller_test.go index 14edff1..0eeeaeb 100644 --- a/internal/controller/openstackcluster_controller_test.go +++ b/internal/controller/openstackcluster_controller_test.go @@ -105,9 +105,9 @@ func getClusterOrNil(t *testing.T, c client.Client, name, namespace string) *inf return &cluster } -// ── US8.1: Ajouter un finalizer ─────────────────────────────────────────────── +// ── US8.1: Add a finalizer ─────────────────────────────────────────────────── -// Scenario: Cluster sans deletionTimestamp et sans finalizer → finalizer ajouté +// Scenario: Cluster without deletionTimestamp and without finalizer → finalizer added func TestReconcile_AddsFinalizer_WhenNotPresent(t *testing.T) { cluster := newCluster("mycluster", "default") r, c := newReconciler(nil, cluster) @@ -125,7 +125,7 @@ func TestReconcile_AddsFinalizer_WhenNotPresent(t *testing.T) { } } -// Scenario: Cluster avec finalizer déjà présent → aucun changement d'état +// Scenario: Cluster with finalizer already present → no state change func TestReconcile_FinalizerAlreadyPresent_Idempotent(t *testing.T) { cluster := newCluster("mycluster", "default", withFinalizer) r, c := newReconciler(nil, cluster) @@ -143,9 +143,9 @@ func TestReconcile_FinalizerAlreadyPresent_Idempotent(t *testing.T) { } } -// ── US8.2: Nom du cluster depuis le label ou metadata.name ─────────────────── +// ── US8.2: Cluster name from label or metadata.name ────────────────────────── -// Scenario: Label cluster.x-k8s.io/cluster-name présent → nom du label utilisé +// Scenario: Label cluster.x-k8s.io/cluster-name present → label name used func TestReconcile_ClusterName_FromLabel(t *testing.T) { cluster := newCluster("mycluster-openstack", "default", withFinalizer, @@ -169,7 +169,7 @@ func TestReconcile_ClusterName_FromLabel(t *testing.T) { } } -// Scenario: Label absent → metadata.name utilisé +// Scenario: Label absent → metadata.name used func TestReconcile_ClusterName_FallsBackToMetadataName(t *testing.T) { cluster := newCluster("mycluster", "default", withFinalizer, withDeletionTimestamp) secret := newSecret("cloud-credentials", "default") @@ -189,9 +189,9 @@ func TestReconcile_ClusterName_FallsBackToMetadataName(t *testing.T) { } } -// ── US8.3: Supprimer le finalizer après nettoyage réussi ───────────────────── +// ── US8.3: Remove the finalizer after successful cleanup ───────────────────── -// Scenario: Purge réussie → finalizer retiré +// Scenario: Successful purge → finalizer removed func TestReconcile_RemovesFinalizer_AfterSuccessfulPurge(t *testing.T) { cluster := newCluster("mycluster", "default", withFinalizer, withDeletionTimestamp) secret := newSecret("cloud-credentials", "default") @@ -209,7 +209,7 @@ func TestReconcile_RemovesFinalizer_AfterSuccessfulPurge(t *testing.T) { } } -// Scenario: Finalizer absent au moment de la suppression → pas de purge +// Scenario: Finalizer absent at deletion time → no purge func TestReconcile_SkipsCleanup_WhenFinalizerAbsent(t *testing.T) { // Use a non-janitor finalizer to keep the cluster alive in the fake client // when we call Delete (which sets DeletionTimestamp instead of deleting immediately). @@ -236,9 +236,9 @@ func TestReconcile_SkipsCleanup_WhenFinalizerAbsent(t *testing.T) { } } -// ── US8.4: Mécanisme de retry via annotation ────────────────────────────────── +// ── US8.4: Retry mechanism via annotation ──────────────────────────────────── -// Scenario: Erreur lors de la purge → annotation retry posée, Reconcile retourne nil +// Scenario: Error during purge → retry annotation set, Reconcile returns nil func TestReconcile_AnnotatesRetry_OnPurgeError(t *testing.T) { cluster := newCluster("mycluster", "default", withFinalizer, withDeletionTimestamp) secret := newSecret("cloud-credentials", "default") @@ -260,7 +260,7 @@ func TestReconcile_AnnotatesRetry_OnPurgeError(t *testing.T) { } } -// Scenario: Cluster supprimé entre l'erreur de purge et l'annotation retry → NotFound ignoré +// Scenario: Cluster deleted between purge error and retry annotation → NotFound ignored func TestReconcile_IgnoresNotFound_WhenClusterDeletedDuringRetry(t *testing.T) { cluster := newCluster("mycluster", "default", withFinalizer, withDeletionTimestamp) secret := newSecret("cloud-credentials", "default") diff --git a/internal/openstack/cloud_test.go b/internal/openstack/cloud_test.go index 608fb63..91e9bdb 100644 --- a/internal/openstack/cloud_test.go +++ b/internal/openstack/cloud_test.go @@ -167,14 +167,14 @@ func endpoint(iface, regionID, url string) map[string]any { } } -// ── US1.1: Authentification via Application Credential v3 ────────────────── - -// Scenario: Authentification réussie -// Given un clouds.yaml avec auth_type "v3applicationcredential" -// And un application_credential_id et application_credential_secret valides -// When l'opérateur initialise la connexion OpenStack -// Then un token X-Auth-Token est obtenu depuis Keystone -// And le catalogue de services est chargé +// ── US1.1: Authentication via Application Credential v3 ──────────────────── + +// Scenario: Successful authentication +// Given a clouds.yaml with auth_type "v3applicationcredential" +// And a valid application_credential_id and application_credential_secret +// When the operator initialises the OpenStack connection +// Then an X-Auth-Token is obtained from Keystone +// And the service catalog is loaded func TestAuthenticate_SuccessfulAuthentication(t *testing.T) { ks := newKeystoneServer(t) ks.catalog = catalogWith( @@ -197,10 +197,10 @@ func TestAuthenticate_SuccessfulAuthentication(t *testing.T) { } } -// Scenario: Authentification avec un type non supporté -// Given un clouds.yaml avec auth_type "password" -// When l'opérateur tente de créer un client Cloud -// Then une erreur UnsupportedAuthTypeError est levée +// Scenario: Authentication with unsupported type +// Given a clouds.yaml with auth_type "password" +// When the operator attempts to create a Cloud client +// Then an UnsupportedAuthTypeError is raised func TestAuthenticate_UnsupportedAuthType(t *testing.T) { ks := newKeystoneServer(t) clouds := buildCloudsYAML(ks.URL, "password") @@ -219,7 +219,7 @@ func TestAuthenticate_UnsupportedAuthType(t *testing.T) { } } -// Scenario: Cloud manquant dans clouds.yaml +// Scenario: Cloud missing in clouds.yaml func TestAuthenticate_CloudNotFound(t *testing.T) { ks := newKeystoneServer(t) clouds := buildCloudsYAML(ks.URL, "v3applicationcredential") @@ -231,7 +231,7 @@ func TestAuthenticate_CloudNotFound(t *testing.T) { } } -// Scenario: clouds.yaml invalide (YAML malformé) +// Scenario: Invalid clouds.yaml (malformed YAML) func TestAuthenticate_InvalidCloudsYAML(t *testing.T) { _, err := openstack.Authenticate(context.Background(), "not: valid: yaml: :", "openstack", "") @@ -240,14 +240,14 @@ func TestAuthenticate_InvalidCloudsYAML(t *testing.T) { } } -// ── US1.3: Gestion d'un credential révoqué ou invalide ───────────────────── +// ── US1.3: Revoked or invalid credential handling ─────────────────────────── -// Scenario: Application credential supprimé avant la purge (token request → 404) -// Given un cluster en cours de suppression -// And l'application credential a déjà été supprimé -// When l'opérateur tente de s'authentifier -// Then is_authenticated retourne false -// And aucune erreur fatale n'est levée +// Scenario: Application credential deleted before purge (token request → 404) +// Given a cluster being deleted +// And the application credential has already been deleted +// When the operator attempts to authenticate +// Then is_authenticated returns false +// And no fatal error is raised func TestAuthenticate_DeletedApplicationCredential_Returns404(t *testing.T) { ks := newKeystoneServer(t) ks.tokenStatus = http.StatusNotFound @@ -263,9 +263,9 @@ func TestAuthenticate_DeletedApplicationCredential_Returns404(t *testing.T) { } } -// Scenario: Token request échoue avec une erreur non-404 -// When l'opérateur tente de s'authentifier -// Then l'erreur est propagée +// Scenario: Token request fails with a non-404 error +// When the operator attempts to authenticate +// Then the error is propagated func TestAuthenticate_TokenRequestFails_NonFatal(t *testing.T) { for _, code := range []int{http.StatusUnauthorized, http.StatusForbidden, http.StatusInternalServerError} { code := code @@ -283,11 +283,11 @@ func TestAuthenticate_TokenRequestFails_NonFatal(t *testing.T) { } } -// Scenario: Catalogue retourne 404 -// Given une URL Keystone valide mais le catalogue retourne 404 -// When l'opérateur charge le catalogue -// Then is_authenticated retourne false -// And aucune erreur fatale n'est levée +// Scenario: Catalog returns 404 +// Given a valid Keystone URL but the catalog returns 404 +// When the operator loads the catalog +// Then is_authenticated returns false +// And no fatal error is raised func TestAuthenticate_CatalogReturns404(t *testing.T) { ks := newKeystoneServer(t) ks.catalogStatus = http.StatusNotFound @@ -303,13 +303,13 @@ func TestAuthenticate_CatalogReturns404(t *testing.T) { } } -// ── US1.2: Filtrage du catalogue par interface et région ──────────────────── +// ── US1.2: Catalog filtering by interface and region ──────────────────────── -// Scenario: Endpoint sélectionné selon l'interface configurée -// Given un catalogue avec des endpoints "public" et "internal" -// And l'interface configurée est "public" -// When le catalogue est chargé -// Then seuls les endpoints "public" sont retenus +// Scenario: Endpoint selected by configured interface +// Given a catalog with "public" and "internal" endpoints +// And the configured interface is "public" +// When the catalog is loaded +// Then only "public" endpoints are retained func TestAuthenticate_FiltersByInterface_Public(t *testing.T) { ks := newKeystoneServer(t) ks.catalog = catalogWith( @@ -341,7 +341,7 @@ func TestAuthenticate_FiltersByInterface_Public(t *testing.T) { } } -// Scenario: Endpoint sélectionné selon l'interface configurée (internal) +// Scenario: Endpoint selected by configured interface (internal) func TestAuthenticate_FiltersByInterface_Internal(t *testing.T) { ks := newKeystoneServer(t) ks.catalog = catalogWith( @@ -367,11 +367,11 @@ func TestAuthenticate_FiltersByInterface_Internal(t *testing.T) { } } -// Scenario: Endpoint sélectionné selon la région configurée -// Given un catalogue avec des endpoints pour "RegionOne" et "RegionTwo" -// And la région configurée est "RegionOne" -// When le catalogue est chargé -// Then seuls les endpoints de "RegionOne" sont retenus +// Scenario: Endpoint selected by configured region +// Given a catalog with endpoints for "RegionOne" and "RegionTwo" +// And the configured region is "RegionOne" +// When the catalog is loaded +// Then only "RegionOne" endpoints are retained func TestAuthenticate_FiltersByRegion(t *testing.T) { ks := newKeystoneServer(t) ks.catalog = catalogWith( @@ -397,11 +397,11 @@ func TestAuthenticate_FiltersByRegion(t *testing.T) { } } -// Scenario: Aucune région configurée -// Given un catalogue avec des endpoints dans plusieurs régions -// And aucune région n'est configurée -// When le catalogue est chargé -// Then le premier endpoint correspondant à l'interface est retenu pour chaque service +// Scenario: No region configured +// Given a catalog with endpoints in multiple regions +// And no region is configured +// When the catalog is loaded +// Then the first endpoint matching the interface is retained for each service func TestAuthenticate_NoRegionFilter_AcceptsAllRegions(t *testing.T) { ks := newKeystoneServer(t) ks.catalog = catalogWith( @@ -427,7 +427,7 @@ func TestAuthenticate_NoRegionFilter_AcceptsAllRegions(t *testing.T) { } } -// Scenario: Catalogue avec plusieurs services +// Scenario: Catalog with multiple services func TestAuthenticate_MultipleServicesInCatalog(t *testing.T) { ks := newKeystoneServer(t) ks.catalog = catalogWith( @@ -455,7 +455,7 @@ func TestAuthenticate_MultipleServicesInCatalog(t *testing.T) { } } -// Scenario: Catalogue vide → non authentifié +// Scenario: Empty catalog → unauthenticated func TestAuthenticate_EmptyCatalog_NotAuthenticated(t *testing.T) { ks := newKeystoneServer(t) ks.catalog = catalogWith() // no services @@ -471,13 +471,13 @@ func TestAuthenticate_EmptyCatalog_NotAuthenticated(t *testing.T) { } } -// ── US1.4: Support des certificats CA personnalisés ───────────────────────── +// ── US1.4: Custom CA certificate support ──────────────────────────────────── -// Scenario: CA fourni dans le secret Kubernetes -// Given un secret Kubernetes contenant une entrée "cacert" -// When l'opérateur initialise le transport TLS -// Then le CA est chargé dans le contexte SSL -// And les appels HTTPS vers OpenStack utilisent ce CA pour la vérification +// Scenario: CA provided in the Kubernetes secret +// Given a Kubernetes secret containing a "cacert" entry +// When the operator initialises the TLS transport +// Then the CA is loaded into the SSL context +// And HTTPS calls to OpenStack use this CA for verification func TestAuthenticate_WithCustomCACert(t *testing.T) { ks := newTLSKeystoneServer(t) ks.catalog = catalogWith( @@ -509,7 +509,7 @@ func TestAuthenticate_WithCustomCACert(t *testing.T) { } } -// Scenario: Pas de CA fourni → CA système utilisé (connexion HTTP doit fonctionner) +// Scenario: No CA provided → system CA used (plain HTTP connection must work) func TestAuthenticate_NoCACert_HTTPServerWorks(t *testing.T) { ks := newKeystoneServer(t) // plain HTTP ks.catalog = catalogWith( @@ -529,7 +529,7 @@ func TestAuthenticate_NoCACert_HTTPServerWorks(t *testing.T) { } } -// Scenario: CA fourni mais invalide → TLS doit échouer +// Scenario: CA provided but invalid → TLS must fail func TestAuthenticate_InvalidCACert_TLSFails(t *testing.T) { ks := newTLSKeystoneServer(t) @@ -544,7 +544,7 @@ func TestAuthenticate_InvalidCACert_TLSFails(t *testing.T) { // ── AppCredentialID ───────────────────────────────────────────────────────── -// Scenario: Extraction de l'ID d'application credential depuis clouds.yaml +// Scenario: Extracting application credential ID from clouds.yaml func TestAppCredentialID_Found(t *testing.T) { clouds := ` clouds: @@ -564,7 +564,7 @@ clouds: } } -// Scenario: Cloud absent → erreur +// Scenario: Cloud absent → error func TestAppCredentialID_CloudNotFound(t *testing.T) { clouds := ` clouds: diff --git a/internal/openstack/resources_test.go b/internal/openstack/resources_test.go index bb599a6..6386b29 100644 --- a/internal/openstack/resources_test.go +++ b/internal/openstack/resources_test.go @@ -286,11 +286,11 @@ func lbKubeName(cluster, suffix string) string { return fmt.Sprintf("kube_service_%s_%s", cluster, suffix) } -// ── US2.1: Identifier les Floating IPs d'un cluster ────────────────────────── +// ── US2.1: Identify Floating IPs of a cluster ──────────────────────────────── -// Scenario: FIP appartenant au cluster → incluse dans la suppression -// Scenario: FIP d'un autre cluster → exclue -// Scenario: FIP sans description Kubernetes → exclue +// Scenario: FIP belonging to the cluster → included in deletion +// Scenario: FIP from another cluster → excluded +// Scenario: FIP without Kubernetes description → excluded func TestDeleteFloatingIPs_Filtering(t *testing.T) { tests := []struct { name string @@ -347,7 +347,7 @@ func TestDeleteFloatingIPs_Filtering(t *testing.T) { } } -// Scenario: FIPs de plusieurs clusters → seules celles du bon cluster sont supprimées +// Scenario: FIPs from multiple clusters → only those matching the target cluster are deleted func TestDeleteFloatingIPs_MultipleIPsPartialMatch(t *testing.T) { srv := newNetworkTestServer(t) fips := []fipRecord{ @@ -378,12 +378,12 @@ func TestDeleteFloatingIPs_MultipleIPsPartialMatch(t *testing.T) { } } -// ── US2.2: Supprimer les Floating IPs ──────────────────────────────────────── +// ── US2.2: Delete Floating IPs ─────────────────────────────────────────────── -// Scenario: Suppression réussie -// Given une FIP appartenant au cluster "mycluster" -// When la purge des FIPs est déclenchée -// Then la FIP est supprimée via l'API Neutron +// Scenario: Successful deletion +// Given a FIP belonging to cluster "mycluster" +// When the FIP purge is triggered +// Then the FIP is deleted via the Neutron API func TestDeleteFloatingIPs_SuccessfulDeletion(t *testing.T) { srv := newNetworkTestServer(t) fips := []fipRecord{ @@ -415,10 +415,10 @@ func TestDeleteFloatingIPs_SuccessfulDeletion(t *testing.T) { } } -// Scenario: Erreur HTTP 400 lors de la suppression -// Then un warning est émis -// And la suppression continue pour les autres FIPs -// And vérification déclenchée (check_fips = true) +// Scenario: HTTP 400 error during deletion +// Then a warning is emitted +// And deletion continues for other FIPs +// And verification is triggered (check_fips = true) func TestDeleteFloatingIPs_TransientError400_ContinuesAndVerifies(t *testing.T) { srv := newNetworkTestServer(t) fips := []fipRecord{ @@ -451,7 +451,7 @@ func TestDeleteFloatingIPs_TransientError400_ContinuesAndVerifies(t *testing.T) } } -// Scenario: Erreur HTTP 409 (Conflict) → même comportement que 400 +// Scenario: HTTP 409 error (Conflict) → same behaviour as 400 func TestDeleteFloatingIPs_TransientError409_Continues(t *testing.T) { srv := newNetworkTestServer(t) fips := []fipRecord{ @@ -466,7 +466,7 @@ func TestDeleteFloatingIPs_TransientError409_Continues(t *testing.T) { } } -// Scenario: Erreur HTTP 500 → exception propagée +// Scenario: HTTP 500 error → exception propagated func TestDeleteFloatingIPs_HTTP500_PropagatesError(t *testing.T) { srv := newNetworkTestServer(t) fips := []fipRecord{ @@ -483,8 +483,8 @@ func TestDeleteFloatingIPs_HTTP500_PropagatesError(t *testing.T) { } } -// Scenario: FIPs toujours présentes après suppression → erreur retournée -// (le contrôleur réessaiera via l'annotation retry) +// Scenario: FIPs still present after deletion → error returned +// (the controller will retry via the retry annotation) func TestDeleteFloatingIPs_StillPresentAfterDeletion_ReturnsError(t *testing.T) { srv := newNetworkTestServer(t) fip := fipRecord{ID: "fip-persistent", Description: fipDesc("mycluster")} @@ -502,7 +502,7 @@ func TestDeleteFloatingIPs_StillPresentAfterDeletion_ReturnsError(t *testing.T) } } -// Scenario: Aucune FIP correspondante → pas de suppression, pas de vérification +// Scenario: No matching FIP → no deletion, no verification func TestDeleteFloatingIPs_NothingToDelete_NoVerification(t *testing.T) { srv := newNetworkTestServer(t) srv.fipLists = [][]fipRecord{{}} // empty list @@ -553,13 +553,13 @@ func TestDeleteFloatingIPs_NoNetworkEndpoint_ReturnsCatalogError(t *testing.T) { } } -// ── Epic 3: Nettoyage des Load Balancers Octavia ────────────────────────────── +// ── Epic 3: Octavia Load Balancer Cleanup ───────────────────────────────────── -// ── US3.1: Identifier les Load Balancers Kubernetes ────────────────────────── +// ── US3.1: Identify Kubernetes Load Balancers ───────────────────────────────── -// Scenario: LB appartenant au cluster → inclus dans la suppression -// Scenario: LB d'un autre cluster → exclu -// Scenario: LB sans préfixe kube_service → exclu +// Scenario: LB belonging to the cluster → included in deletion +// Scenario: LB from another cluster → excluded +// Scenario: LB without kube_service prefix → excluded func TestDeleteLoadBalancers_Filtering(t *testing.T) { tests := []struct { name string @@ -609,9 +609,9 @@ func TestDeleteLoadBalancers_Filtering(t *testing.T) { } } -// ── US3.2: Erreur HTTP lors du listing (PR #261) ────────────────────────────── +// ── US3.2: HTTP error during listing (PR #261) ─────────────────────────────── -// Scenario: Erreur HTTP lors du listing des LBs → log ERROR, pas d'exception +// Scenario: HTTP error during LB listing → ERROR log, no exception func TestDeleteLoadBalancers_ListError_LogsAndSkips(t *testing.T) { srv := newLBTestServer(t) srv.listStatusOverride = http.StatusInternalServerError @@ -630,9 +630,9 @@ func TestDeleteLoadBalancers_ListError_LogsAndSkips(t *testing.T) { } } -// ── US3.3: Supprimer les Load Balancers en cascade ─────────────────────────── +// ── US3.3: Delete Load Balancers with cascade ───────────────────────────────── -// Scenario: Suppression réussie avec cascade=true +// Scenario: Successful deletion with cascade=true func TestDeleteLoadBalancers_SuccessfulDeletion(t *testing.T) { srv := newLBTestServer(t) lbs := []lbRecord{ @@ -664,7 +664,7 @@ func TestDeleteLoadBalancers_SuccessfulDeletion(t *testing.T) { } } -// Scenario: DELETE émis avec cascade=true +// Scenario: DELETE issued with cascade=true func TestDeleteLoadBalancers_CascadeDelete(t *testing.T) { srv := newLBTestServer(t) srv.lbLists = [][]lbRecord{ @@ -691,7 +691,7 @@ func TestDeleteLoadBalancers_CascadeDelete(t *testing.T) { } } -// Scenario: Erreur HTTP 400 lors de la suppression → warning, continue, vérification déclenchée +// Scenario: HTTP 400 error during deletion → warning, continues, verification triggered func TestDeleteLoadBalancers_TransientError400_ContinuesAndVerifies(t *testing.T) { srv := newLBTestServer(t) lbs := []lbRecord{ @@ -721,7 +721,7 @@ func TestDeleteLoadBalancers_TransientError400_ContinuesAndVerifies(t *testing.T } } -// Scenario: Erreur HTTP 500 → exception propagée +// Scenario: HTTP 500 error → exception propagated func TestDeleteLoadBalancers_HTTP500_PropagatesError(t *testing.T) { srv := newLBTestServer(t) srv.lbLists = [][]lbRecord{ @@ -737,7 +737,7 @@ func TestDeleteLoadBalancers_HTTP500_PropagatesError(t *testing.T) { } } -// Scenario: LBs toujours présents après suppression → erreur retournée +// Scenario: LBs still present after deletion → error returned func TestDeleteLoadBalancers_StillPresentAfterDeletion_ReturnsError(t *testing.T) { srv := newLBTestServer(t) lb := lbRecord{ID: "lb-persistent", Name: lbKubeName("mycluster", "svc")} @@ -754,7 +754,7 @@ func TestDeleteLoadBalancers_StillPresentAfterDeletion_ReturnsError(t *testing.T } } -// Scenario: Aucun LB correspondant → pas de suppression, pas de vérification +// Scenario: No matching LB → no deletion, no verification func TestDeleteLoadBalancers_NothingToDelete_NoVerification(t *testing.T) { srv := newLBTestServer(t) srv.lbLists = [][]lbRecord{{}} @@ -911,7 +911,7 @@ func sgDesc(cluster string) string { return fmt.Sprintf("Security Group for Service LoadBalancer in cluster %s", cluster) } -// Scenario: Pas d'endpoint "load-balancer" dans le catalogue → retour nil (LBs ignorés) +// Scenario: No "load-balancer" endpoint in catalog → return nil (LBs skipped) func TestDeleteLoadBalancers_NoLoadBalancerEndpoint_Skips(t *testing.T) { ks := newKeystoneServer(t) ks.catalog = catalogWith( @@ -932,13 +932,13 @@ func TestDeleteLoadBalancers_NoLoadBalancerEndpoint_Skips(t *testing.T) { } } -// ── Epic 4: Nettoyage des Security Groups ──────────────────────────────────── +// ── Epic 4: Security Group Cleanup ─────────────────────────────────────────── -// ── US4.1: Identifier les Security Groups d'un cluster ─────────────────────── +// ── US4.1: Identify Security Groups of a cluster ───────────────────────────── -// Scenario: SG appartenant au cluster → inclus dans la suppression -// Scenario: SG d'un autre cluster → exclu -// Scenario: Description ne correspondant pas → exclu +// Scenario: SG belonging to the cluster → included in deletion +// Scenario: SG from another cluster → excluded +// Scenario: Non-matching description → excluded func TestDeleteSecurityGroups_Filtering(t *testing.T) { tests := []struct { name string @@ -994,9 +994,9 @@ func TestDeleteSecurityGroups_Filtering(t *testing.T) { } } -// ── US4.2: Supprimer les Security Groups ────────────────────────────────────── +// ── US4.2: Delete Security Groups ──────────────────────────────────────────── -// Scenario: Suppression réussie +// Scenario: Successful deletion func TestDeleteSecurityGroups_SuccessfulDeletion(t *testing.T) { srv := newSGTestServer(t) sgs := []sgRecord{ @@ -1028,7 +1028,7 @@ func TestDeleteSecurityGroups_SuccessfulDeletion(t *testing.T) { } } -// Scenario: SG encore utilisé (HTTP 409) → warning, continue, vérification déclenchée +// Scenario: SG still in use (HTTP 409) → warning, continues, verification triggered func TestDeleteSecurityGroups_TransientError409_ContinuesAndVerifies(t *testing.T) { srv := newSGTestServer(t) sgs := []sgRecord{ @@ -1059,7 +1059,7 @@ func TestDeleteSecurityGroups_TransientError409_ContinuesAndVerifies(t *testing. } } -// Scenario: Erreur HTTP 500 → exception propagée +// Scenario: HTTP 500 error → exception propagated func TestDeleteSecurityGroups_HTTP500_PropagatesError(t *testing.T) { srv := newSGTestServer(t) srv.sgLists = [][]sgRecord{ @@ -1075,7 +1075,7 @@ func TestDeleteSecurityGroups_HTTP500_PropagatesError(t *testing.T) { } } -// Scenario: SGs toujours présents après suppression → erreur retournée +// Scenario: SGs still present after deletion → error returned func TestDeleteSecurityGroups_StillPresentAfterDeletion_ReturnsError(t *testing.T) { srv := newSGTestServer(t) sg := sgRecord{ID: "sg-persistent", Description: sgDesc("mycluster")} @@ -1092,7 +1092,7 @@ func TestDeleteSecurityGroups_StillPresentAfterDeletion_ReturnsError(t *testing. } } -// Scenario: Aucun SG correspondant → pas de suppression, pas de vérification +// Scenario: No matching SG → no deletion, no verification func TestDeleteSecurityGroups_NothingToDelete_NoVerification(t *testing.T) { srv := newSGTestServer(t) srv.sgLists = [][]sgRecord{{}} @@ -1241,14 +1241,14 @@ clouds: return session } -// ── Epic 5: Gestion des Volumes Cinder ─────────────────────────────────────── +// ── Epic 5: Cinder Volume Management ───────────────────────────────────────── -// ── US5.1: Identifier les volumes d'un cluster ─────────────────────────────── +// ── US5.1: Identify volumes of a cluster ───────────────────────────────────── -// Scenario: Volume du bon cluster sans keep → supprimé -// Scenario: Volume avec keep=true → conservé -// Scenario: Volume d'un autre cluster → exclu -// Scenario: Volume sans métadonnée CSI → exclu +// Scenario: Volume from the correct cluster without keep → deleted +// Scenario: Volume with keep=true → kept +// Scenario: Volume from another cluster → excluded +// Scenario: Volume without CSI metadata → excluded func TestDeleteVolumes_Filtering(t *testing.T) { tests := []struct { name string @@ -1307,7 +1307,7 @@ func TestDeleteVolumes_Filtering(t *testing.T) { } } -// Scenario: Suppression réussie de plusieurs volumes +// Scenario: Successful deletion of multiple volumes func TestDeleteVolumes_SuccessfulDeletion(t *testing.T) { srv := newCinderTestServer(t) vols := []cinderVolumeRecord{ @@ -1339,7 +1339,7 @@ func TestDeleteVolumes_SuccessfulDeletion(t *testing.T) { } } -// Scenario: Erreur HTTP 409 (transiente) → warning, continue, vérification déclenchée +// Scenario: Transient HTTP 409 error → warning, continues, verification triggered func TestDeleteVolumes_TransientError409_ContinuesAndVerifies(t *testing.T) { srv := newCinderTestServer(t) vols := []cinderVolumeRecord{ @@ -1369,7 +1369,7 @@ func TestDeleteVolumes_TransientError409_ContinuesAndVerifies(t *testing.T) { } } -// Scenario: Erreur HTTP 500 → exception propagée +// Scenario: HTTP 500 error → exception propagated func TestDeleteVolumes_HTTP500_PropagatesError(t *testing.T) { srv := newCinderTestServer(t) srv.volumeLists = [][]cinderVolumeRecord{ @@ -1385,7 +1385,7 @@ func TestDeleteVolumes_HTTP500_PropagatesError(t *testing.T) { } } -// Scenario: Volume toujours présent après suppression → erreur retournée +// Scenario: Volume still present after deletion → error returned func TestDeleteVolumes_StillPresentAfterDeletion_ReturnsError(t *testing.T) { srv := newCinderTestServer(t) vol := cinderVolumeRecord{ @@ -1405,7 +1405,7 @@ func TestDeleteVolumes_StillPresentAfterDeletion_ReturnsError(t *testing.T) { } } -// Scenario: Aucun volume correspondant → pas de suppression, pas de vérification +// Scenario: No matching volume → no deletion, no verification func TestDeleteVolumes_NothingToDelete_NoVerification(t *testing.T) { srv := newCinderTestServer(t) srv.volumeLists = [][]cinderVolumeRecord{{}} @@ -1548,12 +1548,12 @@ clouds: return session } -// ── Epic 6: Gestion des Snapshots Cinder ───────────────────────────────────── +// ── Epic 6: Cinder Snapshot Management ─────────────────────────────────────── -// ── US6.1: Identifier et supprimer les snapshots d'un cluster ──────────────── +// ── US6.1: Identify and delete snapshots of a cluster ──────────────────────── -// Scenario: Snapshot du bon cluster → supprimé -// Scenario: Snapshot d'un autre cluster → exclu +// Scenario: Snapshot from the correct cluster → deleted +// Scenario: Snapshot from another cluster → excluded func TestDeleteSnapshots_Filtering(t *testing.T) { tests := []struct { name string @@ -1603,7 +1603,7 @@ func TestDeleteSnapshots_Filtering(t *testing.T) { } } -// Scenario: Suppression réussie de plusieurs snapshots +// Scenario: Successful deletion of multiple snapshots func TestDeleteSnapshots_SuccessfulDeletion(t *testing.T) { srv := newSnapshotTestServer(t) snaps := []cinderVolumeRecord{ @@ -1635,7 +1635,7 @@ func TestDeleteSnapshots_SuccessfulDeletion(t *testing.T) { } } -// Scenario: Erreur HTTP 409 (transiente) → warning, continue, vérification déclenchée +// Scenario: Transient HTTP 409 error → warning, continues, verification triggered func TestDeleteSnapshots_TransientError409_ContinuesAndVerifies(t *testing.T) { srv := newSnapshotTestServer(t) snaps := []cinderVolumeRecord{ @@ -1665,7 +1665,7 @@ func TestDeleteSnapshots_TransientError409_ContinuesAndVerifies(t *testing.T) { } } -// Scenario: Erreur HTTP 500 → exception propagée +// Scenario: HTTP 500 error → exception propagated func TestDeleteSnapshots_HTTP500_PropagatesError(t *testing.T) { srv := newSnapshotTestServer(t) srv.snapshotLists = [][]cinderVolumeRecord{ @@ -1681,7 +1681,7 @@ func TestDeleteSnapshots_HTTP500_PropagatesError(t *testing.T) { } } -// Scenario: Snapshot toujours présent après suppression → erreur retournée +// Scenario: Snapshot still present after deletion → error returned func TestDeleteSnapshots_StillPresentAfterDeletion_ReturnsError(t *testing.T) { srv := newSnapshotTestServer(t) snap := cinderVolumeRecord{ @@ -1800,11 +1800,11 @@ clouds: return session, cloudsYAML } -// ── Epic 7: Gestion des Application Credentials ─────────────────────────────── +// ── Epic 7: Application Credential Management ───────────────────────────────── -// ── US7.1: Supprimer l'Application Credential OpenStack ────────────────────── +// ── US7.1: Delete the OpenStack Application Credential ─────────────────────── -// Scenario: Suppression autorisée → HTTP 204, retour nil +// Scenario: Deletion authorised → HTTP 204, return nil func TestDeleteAppCredential_SuccessfulDeletion(t *testing.T) { srv := newIdentityTestServer(t) session, cloudsYAML := srv.authenticate(t) @@ -1822,7 +1822,7 @@ func TestDeleteAppCredential_SuccessfulDeletion(t *testing.T) { } } -// Scenario: Application Credential déjà supprimé → HTTP 404, retour nil +// Scenario: Application Credential already deleted → HTTP 404, return nil func TestDeleteAppCredential_AlreadyDeleted_404(t *testing.T) { srv := newIdentityTestServer(t) srv.deleteStatus = http.StatusNotFound @@ -1833,7 +1833,7 @@ func TestDeleteAppCredential_AlreadyDeleted_404(t *testing.T) { } } -// Scenario: Application Credential non supprimable (403) → warning, retour nil +// Scenario: Application Credential cannot be deleted (403) → warning, return nil func TestDeleteAppCredential_Forbidden_403(t *testing.T) { srv := newIdentityTestServer(t) srv.deleteStatus = http.StatusForbidden @@ -1844,7 +1844,7 @@ func TestDeleteAppCredential_Forbidden_403(t *testing.T) { } } -// Scenario: Erreur HTTP 500 → erreur propagée +// Scenario: HTTP 500 error → error propagated func TestDeleteAppCredential_HTTP500_PropagatesError(t *testing.T) { srv := newIdentityTestServer(t) srv.deleteStatus = http.StatusInternalServerError @@ -1856,7 +1856,7 @@ func TestDeleteAppCredential_HTTP500_PropagatesError(t *testing.T) { } } -// Scenario: Pas d'endpoint "identity" dans le catalogue → CatalogError +// Scenario: No "identity" endpoint in catalog → CatalogError func TestDeleteAppCredential_NoIdentityEndpoint_ReturnsCatalogError(t *testing.T) { ks := newKeystoneServer(t) ks.catalog = catalogWith( @@ -1882,7 +1882,7 @@ func TestDeleteAppCredential_NoIdentityEndpoint_ReturnsCatalogError(t *testing.T } } -// Scenario: Aucun snapshot correspondant → pas de suppression, pas de vérification +// Scenario: No matching snapshot → no deletion, no verification func TestDeleteSnapshots_NothingToDelete_NoVerification(t *testing.T) { srv := newSnapshotTestServer(t) srv.snapshotLists = [][]cinderVolumeRecord{{}} diff --git a/internal/openstack/robustness_test.go b/internal/openstack/robustness_test.go index 7e3034e..d1c1709 100644 --- a/internal/openstack/robustness_test.go +++ b/internal/openstack/robustness_test.go @@ -16,7 +16,7 @@ import ( // ── US12.1 : Timeout HTTP ───────────────────────────────────────────────────── -// Scenario: le contexte est déjà annulé → Authenticate retourne une erreur immédiatement +// Scenario: context already cancelled → Authenticate returns an error immediately func TestAuthenticate_ReturnsError_WhenContextAlreadyCancelled(t *testing.T) { // Use a real (if short-lived) server so Authenticate reaches the HTTP call. ks := newKeystoneServer(t) @@ -125,7 +125,7 @@ clouds: return session } -// Scenario: catalog avec "block-storage" → cinderEndpoint résout le bon endpoint +// Scenario: catalog with "block-storage" → cinderEndpoint resolves the correct endpoint func TestCinderEndpoint_FallsBackToBlockStorage(t *testing.T) { srv := newCinderAliasServer(t, "block-storage") session := srv.authenticate(t) @@ -139,7 +139,7 @@ func TestCinderEndpoint_FallsBackToBlockStorage(t *testing.T) { } } -// Scenario: catalog avec "volume" uniquement → cinderEndpoint utilise l'alias legacy +// Scenario: catalog with "volume" only → cinderEndpoint uses the legacy alias func TestCinderEndpoint_FallsBackToVolumeAlias(t *testing.T) { srv := newCinderAliasServer(t, "volume") session := srv.authenticate(t) From 5c6d9158d10be063feeaaa5b9165921e5af58744 Mon Sep 17 00:00:00 2001 From: Mathieu Grzybek Date: Wed, 8 Jul 2026 13:41:51 +0200 Subject: [PATCH 16/27] feat(doc): update README Signed-off-by: Mathieu Grzybek --- README.md | 254 ++++++++++++++++++++++++++++++++++++------------------ 1 file changed, 170 insertions(+), 84 deletions(-) diff --git a/README.md b/README.md index 801a63a..331213c 100644 --- a/README.md +++ b/README.md @@ -3,128 +3,214 @@ `cluster-api-janitor-openstack` is a Kubernetes operator that cleans up resources created in [OpenStack](https://www.openstack.org/) by the [OpenStack Cloud Controller Manager (OCCM)](https://github.com/kubernetes/cloud-provider-openstack/blob/master/docs/openstack-cloud-controller-manager/using-openstack-cloud-controller-manager.md) -and +and the [Cinder CSI plugin](https://github.com/kubernetes/cloud-provider-openstack/blob/master/docs/cinder-csi-plugin/using-cinder-csi-plugin.md) -for Kubernetes clusters created using the -[Cluster API OpenStack infrastructure provider](https://github.com/kubernetes-sigs/cluster-api-provider-openstack). +for Kubernetes clusters created with the +[Cluster API OpenStack infrastructure provider (CAPO)](https://github.com/kubernetes-sigs/cluster-api-provider-openstack). -## Installation +The operator watches `OpenStackCluster` resources and, upon deletion, removes any +dangling OpenStack resources (floating IPs, load balancers, security groups, Cinder +volumes and snapshots, and the application credential) that would otherwise be left +behind after the CAPI cluster is gone. + +## Requirements + +| Tool | Minimum version | +|---|---| +| Go | 1.26 | +| Kubernetes | 1.29 | +| CAPO | 0.14 | +| Helm | 3.x | + +## How it works + +1. When an `OpenStackCluster` is created, the operator adds its finalizer + (`janitor.capi.stackhpc.com`) to the resource. +2. When the `OpenStackCluster` is marked for deletion (`deletionTimestamp` set), + the operator authenticates to OpenStack using the credential referenced by + `spec.identityRef` and deletes all resources tagged with the cluster name. +3. The cluster name is taken from the `cluster.x-k8s.io/cluster-name` label if + present, falling back to `metadata.name`. +4. Once the purge succeeds the finalizer is removed and the `OpenStackCluster` can + be fully deleted. +5. If the purge fails, the operator sets a retry annotation + (`janitor.capi.stackhpc.com/retry`) and returns without error, triggering a + re-reconcile. + +> **Why a finalizer instead of a post-delete job?** +> +> Some OCCM-created load balancers hold references to the cluster network, which +> prevents the Cluster API OpenStack provider from deleting that network. Running +> cleanup *before* the network is torn down (but *after* all machines are gone) +> avoids this deadlock and eliminates any race with a still-running OCCM. + +## Resources cleaned up + +| OpenStack service | Resources | +|---|---| +| Neutron | Floating IPs associated with `LoadBalancer` services | +| Octavia | Load balancers with name prefix `kube_service__` | +| Neutron | Security groups matching the OCCM naming convention | +| Cinder | Volumes provisioned by the Cinder CSI (configurable — see below) | +| Cinder | Snapshots of those volumes | +| Keystone | The application credential used by the cluster (if authorised) | + +## Configuration + +### Volume deletion policy + +Cinder volumes are deleted by default. This can be changed at two levels: + +**Operator-wide default** (via Helm): + +```sh +helm upgrade ... --set defaultVolumesPolicy=keep +``` + +**Per-cluster override** (annotation on `OpenStackCluster`): + +```yaml +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: OpenStackCluster +metadata: + name: my-cluster + annotations: + janitor.capi.stackhpc.com/volumes-policy: "keep" # or "delete" +``` + +> Any value other than `delete` means volumes will be kept. + +**Per-volume override** (set directly on the OpenStack volume): + +```sh +openstack volume set --property janitor.capi.azimuth-cloud.com/keep=true +``` + +Any value other than `true` results in the volume being deleted. + +### Retry delay + +When cleanup fails, the operator waits before re-queuing. The default delay is +60 seconds and can be changed via Helm: -`cluster-api-janitor-openstack` can be installed using [Helm](https://helm.sh): +```sh +helm upgrade ... --set retryDefaultDelay=120 +``` + +### Environment variables + +| Variable | Default | Description | +|---|---|---| +| `CAPI_JANITOR_DEFAULT_VOLUMES_POLICY` | `delete` | Operator-wide volume policy | +| `CAPI_JANITOR_RETRY_DEFAULT_DELAY` | `60` | Retry delay in seconds | + +## Installation ```sh helm repo add \ cluster-api-janitor-openstack \ https://azimuth-cloud.github.io/cluster-api-janitor-openstack -# Use the latest version from the main branch helm upgrade \ cluster-api-janitor-openstack \ cluster-api-janitor-openstack/cluster-api-janitor-openstack \ --install ``` -## Tox for unittests and linting +## Development -We use tox to run unit tests and linters across the code. -To run all the checks, including efforts to automatically -fix linting issues, please run: +### Build ```sh -tox +go build ./... ``` -You can run individual unit tests by running: +### Run tests ```sh -tox -e py3 -- +go test ./... ``` -Note, failures on your initial tox run may be automatically -fixed, where possible. So your second tox run may pass. -This way we can run the default tox target in CI. - -## Configuration +The test suite covers 108 unit tests across 4 packages using only the standard +`testing` package and `controller-runtime`'s fake client — no external cluster +required. -`cluster-api-janitor-openstack` will always clean up -[Octavia loadbalancers](https://docs.openstack.org/octavia/latest/), and associated -[floating IPs](https://docs.openstack.org/neutron/latest/), that are created by -the OCCM for `LoadBalancer` services on Cluster API clusters. +### Lint and format -By default, [Cinder volumes](https://docs.openstack.org/cinder/latest/) created by the -Cinder CSI plugin for `PersistentVolumeClaim`s are also cleaned up. However this behaviour -carries a risk of deleting important data, so can be customised in two ways. +```sh +go fmt ./... +go vet ./... +golangci-lint run ./... # config in .golangci.yml +``` -The operator default can be changed to `keep`, meaning that volumes provisioned by the -Cinder CSI plugin will be kept unless overridden by the cluster: +### Makefile targets ```sh -helm upgrade ... --set defaultVolumesPolicy=keep +make help # list all targets +make generate # regenerate DeepCopy methods +make manifests # regenerate CRD/RBAC YAML +make fmt # go fmt +make vet # go vet +make test # go test with race detector +make build # go build ./cmd/main.go ``` -Regardless of the operator default, individual `OpenStackCluster`s can also be annotated -to indicate whether volumes for that cluster should be kept or removed: +## Building the OCI image -```yaml -apiVersion: infrastructure.cluster.x-k8s.io/v1alpha7 -kind: OpenStackCluster -metadata: - name: my-cluster - annotations: - janitor.capi.stackhpc.com/volumes-policy: "keep|delete" +### Docker (local development) + +```sh +docker build -t cluster-api-janitor-openstack:dev . ``` -> **NOTE**: Any value other than `delete` means volumes will be kept. +The Dockerfile uses a multi-stage build: `golang:1.26` builder → +`gcr.io/distroless/static:nonroot` runtime (UID 65532, read-only root FS). -### User-configurable behaviour +### Nix (reproducible, multi-arch + SBOM) -Annotations on the Kubernetes resources are only available to administrators with -access to the Cluster API management cluster's Kubernetes API; therefore, the Janitor -also provides an alternative user-facing mechanism for marking volumes which should -not be deleted during cluster clean up. This is done by adding a property to the -OpenStack volume using: +CI uses `nix-build` for reproducible images and SBOM generation: -``` -openstack volume set --property janitor.capi.azimuth-cloud.com/keep='true' +```sh +# Build the amd64 OCI image +nix-build nix -A image + +# Build the arm64 OCI image (cross-compiled) +nix-build nix -A image-arm64 + +# Generate the CycloneDX SBOM +nix-build nix -A sbom && cat result | python3 -m json.tool | grep bomFormat ``` -Any value other than 'true' will result in the volume being deleted when the workload -cluster is deleted. +> `nix/nixpkgs.nix` pins `nixos-26.05` (required for Go 1.26). +> On the first run, `nix-build nix -A manager` will fail with the real +> `vendorHash` to substitute in `nix/default.nix`. -## How it works +## Observability -`cluster-api-janitor-openstack` watches for `OpenStackCluster`s being created and adds its -own finalizer to them. This prevents the `OpenStackCluster`, and hence the corresponding -Cluster API `Cluster`, from being removed until the finalizer is removed. - -`cluster-api-janitor-openstack` then waits for the `OpenStackCluster` to be deleted -(specifically, it waits for the `deletionTimestamp` to be set, indicating that a deletion -has been requested), at which point it uses the credential from -`OpenStackCluster.spec.identityRef` to remove any dangling resources that were created by -the OCCM or Cinder CSI with the same cluster name as the cluster being deleted. -The cluster name is determined by the `cluster.x-k8s.io/cluster-name` label on the -OpenStackCluster resource, if present. -If the label is not set, the name of the OpenStackCluster resource (`metadata.name`) is -used instead. -Once all the resources have been deleted, the finalizer is removed. - -> **WARNING** -> -> The cluster name of the OCCM and Cinder CSI **must** be set to the `metadata.name` -> of the OpenStackCluster resource, or to the value of the `cluster.x-k8s.io/cluster-name` -> label if it is present on the OpenStackCluster resource. -> -> For instance, the `openstack-cluster` chart from the -> [capi-helm-charts](https://github.com/azimuth-cloud/capi-helm-charts) ensures that this happens -> automatically and sets the OpenStackCluster's `metadata.name` for OCCM and Cinder CSI. - -The advantage of this approach vs. a task that runs before the cluster deletion is started -is that the external resource deletion happens _after_ all the machines have been deleted, -meaning that there is no chance of racing with the OCCM and/or Cinder CSI still running on -the cluster that may continue to try and replace resources that are cleaned up. - -It is not possible to run this cleanup as a post cluster deletion task, because some of the -resources created by the OCCM may actually block cluster deletion completely. For example, -a load-balancer created by the OCCM for a `LoadBalancer` service maintains a port on the cluster -network, meaning that the network cannot be cleaned up by the Cluster API OpenStack provider -and preventing deletion of the cluster. +### Prometheus metrics + +| Metric | Labels | Description | +|---|---|---| +| `capi_janitor_cleanups_total` | `result="success\|failure"` | Total cleanup attempts | + +### Kubernetes events + +| Reason | Type | Emitted when | +|---|---|---| +| `CleanupSucceeded` | Normal | OpenStack purge completed successfully | +| `CleanupFailed` | Warning | OpenStack purge returned an error | + +## Project layout + +``` +cmd/ # Operator entry point +internal/ + controller/ # Reconciler, metrics, config + openstack/ # Authentication, resource discovery & deletion +chart/ # Helm chart + templates/ + tests/ # helm-unittest tests +nix/ # Reproducible OCI build + SBOM (no Flake) +config/ # Kustomize bases (RBAC, manager, Prometheus) +test/e2e/ # End-to-end test suite (Ginkgo) +``` From 3b5a96f5d0e0d8eaa4795b43cc5633ca83e599ec Mon Sep 17 00:00:00 2001 From: Mathieu Grzybek Date: Wed, 8 Jul 2026 14:02:12 +0200 Subject: [PATCH 17/27] chores: remove Python code Signed-off-by: Mathieu Grzybek --- .github/workflows/test-pr.yaml | 8 +- .github/workflows/tox.yaml | 45 -- .gitignore | 17 +- .stestr.conf | 3 - capi_janitor/__init__.py | 0 capi_janitor/openstack/__init__.py | 0 capi_janitor/openstack/openstack.py | 249 --------- capi_janitor/openstack/operator.py | 496 ------------------ capi_janitor/tests/__init__.py | 0 capi_janitor/tests/openstack/__init__.py | 0 .../tests/openstack/test_openstack.py | 280 ---------- capi_janitor/tests/openstack/test_operator.py | 475 ----------------- pyproject.toml | 44 -- requirements.txt | 24 - setup.cfg | 18 - setup.py | 6 - tox.ini | 55 -- 17 files changed, 11 insertions(+), 1709 deletions(-) delete mode 100644 .github/workflows/tox.yaml delete mode 100644 .stestr.conf delete mode 100644 capi_janitor/__init__.py delete mode 100644 capi_janitor/openstack/__init__.py delete mode 100644 capi_janitor/openstack/openstack.py delete mode 100644 capi_janitor/openstack/operator.py delete mode 100644 capi_janitor/tests/__init__.py delete mode 100644 capi_janitor/tests/openstack/__init__.py delete mode 100644 capi_janitor/tests/openstack/test_openstack.py delete mode 100644 capi_janitor/tests/openstack/test_operator.py delete mode 100644 pyproject.toml delete mode 100644 requirements.txt delete mode 100755 setup.cfg delete mode 100755 setup.py delete mode 100644 tox.ini diff --git a/.github/workflows/test-pr.yaml b/.github/workflows/test-pr.yaml index d80461b..9dc5b8a 100644 --- a/.github/workflows/test-pr.yaml +++ b/.github/workflows/test-pr.yaml @@ -25,12 +25,6 @@ concurrency: # TODO(mkjpryor): Change this in the future to use the CAPI management only variation ##### jobs: - # Run the unit tests on every PR, even from external repos - unit_tests: - uses: ./.github/workflows/tox.yaml - with: - ref: ${{ github.event.pull_request.head.sha }} - # Run the chart linting on every PR, even from external repos lint: uses: ./.github/workflows/helm-lint.yaml @@ -39,7 +33,7 @@ jobs: # This job exists so that PRs from outside the main repo are rejected fail_on_remote: - needs: [unit_tests, lint] + needs: [lint] runs-on: ubuntu-latest steps: - name: PR must be from a branch in the azimuth-cloud/cluster-api-janitor-openstack repo diff --git a/.github/workflows/tox.yaml b/.github/workflows/tox.yaml deleted file mode 100644 index db2ae36..0000000 --- a/.github/workflows/tox.yaml +++ /dev/null @@ -1,45 +0,0 @@ -name: Tox unit tests - -on: - workflow_call: - inputs: - ref: - type: string - description: The ref to build. - required: true - -jobs: - build: - name: Tox unit tests and linting - runs-on: ubuntu-24.04 - strategy: - matrix: - python-version: ['3.12','3.13'] - - steps: - - name: Check out the repository - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - ref: ${{ inputs.ref || github.ref }} - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 - with: - python-version: ${{ matrix.python-version }} - - - name: Install dependencies - run: | - python -m pip install --upgrade pip - python -m pip install tox - - - name: Test with tox - run: tox - - - name: Generate coverage reports - run: tox -e cover - - - name: Archive code coverage results - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 - with: - name: "code-coverage-report-${{ matrix.python-version }}" - path: cover/ diff --git a/.gitignore b/.gitignore index bf551e2..af84511 100644 --- a/.gitignore +++ b/.gitignore @@ -1,9 +1,12 @@ -__pycache__ -*.egg-info -.python-version +# Go +/bin/ +*.test +*.out + +# Helm chart/charts/* chart/Chart.lock -# ignore unit test stuff -.stestr/* -.tox/* -.coverage + +# Nix +result +result-* diff --git a/.stestr.conf b/.stestr.conf deleted file mode 100644 index dd7e683..0000000 --- a/.stestr.conf +++ /dev/null @@ -1,3 +0,0 @@ -[DEFAULT] -test_path=./capi_janitor/tests -top_dir=./ diff --git a/capi_janitor/__init__.py b/capi_janitor/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/capi_janitor/openstack/__init__.py b/capi_janitor/openstack/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/capi_janitor/openstack/openstack.py b/capi_janitor/openstack/openstack.py deleted file mode 100644 index 4744aff..0000000 --- a/capi_janitor/openstack/openstack.py +++ /dev/null @@ -1,249 +0,0 @@ -import asyncio -import contextlib -import re -import urllib.parse - -import httpx -from easykube import rest - - -class UnsupportedAuthenticationError(Exception): - """Raised when an unsupported authentication method is used.""" - - def __init__(self, auth_type): - super().__init__(f"unsupported authentication type: {auth_type}") - - -class AuthenticationError(Exception): - """Raised when an unknown authentication error is encountered.""" - - def __init__(self, user): - super().__init__(f"failed to authenticate as user: {user}") - - -class CatalogError(Exception): - """Raised when an unknown catalog service type is requested.""" - - def __init__(self, name): - super().__init__(f"service type {name} not found in OpenStack service catalog") - - -class Auth(httpx.Auth): - """Authenticator class for OpenStack connections.""" - - def __init__( - self, auth_url, application_credential_id, application_credential_secret - ): - self.url = auth_url - self._application_credential_id = application_credential_id - self._application_credential_secret = application_credential_secret - self._token = None - self._user_id = None - self._lock = asyncio.Lock() - - @contextlib.asynccontextmanager - async def _refresh_token(self): - """Context manager to ensure only one request at a time - - triggers a token refresh. - """ - token = self._token - async with self._lock: - # Only yield to the wrapped block if the token has not changed - # in the time it took to acquire the lock - if token == self._token: - yield - - def _build_token_request(self): - return httpx.Request( - "POST", - f"{self.url}/v3/auth/tokens", - json={ - "auth": { - "identity": { - "methods": ["application_credential"], - "application_credential": { - "id": self._application_credential_id, - "secret": self._application_credential_secret, - }, - }, - }, - }, - ) - - def _handle_token_response(self, response): - response.raise_for_status() - self._token = response.headers["X-Subject-Token"] - self._user_id = response.json()["token"]["user"]["id"] - - async def async_auth_flow(self, request): - if self._token is None: - async with self._refresh_token(): - response = yield self._build_token_request() - await response.aread() - self._handle_token_response(response) - request.headers["X-Auth-Token"] = self._token - response = yield request - - -class Resource(rest.Resource): - """Base resource for OpenStack APIs.""" - - def __init__(self, client, name, prefix=None, plural_name=None, singular_name=None): - super().__init__(client, name, prefix) - # Some resources support a /detail endpoint - # In this case, we just want to use the name up to the slash as the plural name - self._plural_name = plural_name or self._name.split("/")[0].replace("-", "_") - # If no singular name is given, assume the name ends in 's' - self._singular_name = singular_name or self._plural_name[:-1] - - @property - def singular_name(self): - return self._singular_name - - def _extract_list(self, response): - # Some resources support a /detail endpoint - # In this case, we just want to use the name up to the slash - return response.json()[self._plural_name] - - def _extract_next_page(self, response): - next_url = next( - ( - link["href"] - for link in response.json().get(f"{self._plural_name}_links", []) - if link["rel"] == "next" - ), - None, - ) - # Sometimes, the returned URLs have http where they should have https - # To mitigate this, we split the URL and return the path and params separately - url = urllib.parse.urlsplit(next_url) - params = urllib.parse.parse_qs(url.query) - return url.path, params - - def _extract_one(self, response): - content_type = response.headers.get("content-type") - if content_type == "application/json": - return response.json()[self._singular_name] - else: - return super()._extract_one(response) - - -class Client(rest.AsyncClient): - """Client for OpenStack APIs.""" - - def __init__(self, /, base_url, prefix=None, **kwargs): - # Extract the path part of the base_url - url = urllib.parse.urlsplit(base_url) - # Initialise the client with the scheme/host - super().__init__(base_url=f"{url.scheme}://{url.netloc}", timeout=60, **kwargs) - # If another prefix is not given, use the path from the base URL as the prefix, - # otherwise combine the prefixes and remove duplicated path sections. - # This ensures things like pagination work nicely without duplicating the prefix - if prefix: - self._prefix = "/".join( - [url.path.rstrip("/"), prefix.lstrip("/").lstrip(url.path)] - ) - else: - self._prefix = url.path - - def __aenter__(self): - # Prevent individual clients from being used in a context manager - raise RuntimeError("clients must be used via a cloud object") - - def resource(self, name, prefix=None, plural_name=None, singular_name=None): - # If an additional prefix is given, combine it with the existing prefix - if prefix: - prefix = "/".join([self._prefix.rstrip("/"), prefix.lstrip("/")]) - else: - prefix = self._prefix - return Resource(self, name, prefix, plural_name, singular_name) - - -class Cloud: - """Object for interacting with OpenStack clouds.""" - - def __init__(self, auth, transport, interface, region=None): - self._auth = auth - self._transport = transport - self._interface = interface - self._endpoints = {} - self._region = region - # A map of api name to client - self._clients = {} - - async def __aenter__(self): - await self._transport.__aenter__() - # Once the transport has been initialised, we can initialise the endpoints - client = Client( - base_url=self._auth.url, auth=self._auth, transport=self._transport - ) - try: - # Use the full auth URL to avoid losing any path prefix (e.g. - # /identity) that Client.__init__ strips from the httpx base_url - # into _prefix, which raw client.get() calls do not prepend. - response = await client.get(f"{self._auth.url}/v3/auth/catalog") - except httpx.HTTPStatusError as exc: - # If the auth fails, we just have an empty app catalog - if exc.response.status_code == 404: - return self - else: - raise - self._endpoints = {} - for entry in response.json()["catalog"]: - for ep in entry.get("endpoints", []): - if ep.get("interface") == self._interface and ( - not self._region or ep.get("region_id") == self._region - ): - self._endpoints[entry["type"]] = ep["url"] - break - return self - - async def __aexit__(self, exc_type, exc_value, traceback): - await self._transport.__aexit__(exc_type, exc_value, traceback) - - @property - def is_authenticated(self): - """Returns True if the cloud is authenticated, False otherwise.""" - return bool(self._endpoints) - - @property - def current_user_id(self): - """The ID of the current user.""" - return self._auth._user_id - - @property - def apis(self): - """The APIs supported by the cloud.""" - return list(self._endpoints.keys()) - - def api_client(self, name, prefix=None): - """Returns a client for the named API.""" - if name not in self._clients: - self._clients[name] = Client( - base_url=self._endpoints[name], - prefix=prefix, - auth=self._auth, - transport=self._transport, - ) - return self._clients[name] - - @classmethod - def from_clouds(cls, clouds, cloud, cacert): - config = clouds["clouds"][cloud] - if config["auth_type"] != "v3applicationcredential": - raise UnsupportedAuthenticationError(config["auth_type"]) - auth_url = re.sub("/v3/?$", "", config["auth"]["auth_url"]) - auth = Auth( - auth_url, - config["auth"]["application_credential_id"], - config["auth"]["application_credential_secret"], - ) - region = config.get("region_name") - # Create a default context using the verification from the config - context = httpx.create_ssl_context(verify=config.get("verify", True)) - # If a cacert was given, load it into the context - if cacert is not None: - context.load_verify_locations(cadata=cacert) - transport = httpx.AsyncHTTPTransport(verify=context) - return cls(auth, transport, config.get("interface", "public"), region) diff --git a/capi_janitor/openstack/operator.py b/capi_janitor/openstack/operator.py deleted file mode 100644 index 1197550..0000000 --- a/capi_janitor/openstack/operator.py +++ /dev/null @@ -1,496 +0,0 @@ -import asyncio -import base64 -import functools -import os -import random -import string - -import easykube -import httpx -import kopf -import yaml - -from . import openstack - -ekconfig: easykube.Configuration -ekclient: easykube.AsyncClient - -CAPO_API_GROUP = "infrastructure.cluster.x-k8s.io" -FINALIZER = "janitor.capi.stackhpc.com" - -VOLUMES_ANNOTATION = "janitor.capi.stackhpc.com/volumes-policy" -VOLUMES_ANNOTATION_DELETE = "delete" -VOLUMES_ANNOTATION_DEFAULT = os.environ.get( - "CAPI_JANITOR_DEFAULT_VOLUMES_POLICY", VOLUMES_ANNOTATION_DELETE -) - -CREDENTIAL_ANNOTATION = "janitor.capi.stackhpc.com/credential-policy" -CREDENTIAL_ANNOTATION_DELETE = "delete" - -RETRY_ANNOTATION = "janitor.capi.stackhpc.com/retry" -RETRY_DEFAULT_DELAY = int(os.environ.get("CAPI_JANITOR_RETRY_DEFAULT_DELAY", "60")) - -# The property on the OpenStack volume resource which, if set to 'true', -# will instruct the Janitor to ignore this volume when cleaning up cluster -# resources. -OPENSTACK_USER_VOLUMES_RECLAIM_PROPERTY = "janitor.capi.azimuth-cloud.com/keep" - - -@kopf.on.startup() -async def on_startup(**kwargs): - global ekconfig - ekconfig = easykube.Configuration.from_environment() - global ekclient - ekclient = ekconfig.async_client() - - -@kopf.on.cleanup() -async def on_cleanup(**kwargs): - """Runs on operator shutdown.""" - # Make sure that the easykube client is shut down properly - await ekclient.aclose() - - -class FinalizerStillPresentError(Exception): - """Raised when a finalizer from another controller is preventing us from - - deleting an appcred. - """ - - def __init__(self, finalizer, cluster): - super().__init__(f"finalizer '{finalizer}' still present for cluster {cluster}") - - -class ResourcesStillPresentError(Exception): - """Raised when cluster resources are still present even after being deleted, - - e.g. while waiting for deletion. - """ - - def __init__(self, resource, cluster): - super().__init__(f"{resource} still present for cluster {cluster}") - - -async def fips_for_cluster(resource, cluster): - """Async iterator for FIPs belonging to the specified cluster.""" - async for fip in resource.list(): - if not fip.description.startswith( - "Floating IP for Kubernetes external service" - ): - continue - if not fip.description.endswith(f"from cluster {cluster}"): - continue - yield fip - - -async def lbs_for_cluster(resource, cluster): - """Async iterator for loadbalancers belonging to the specified cluster.""" - async for lb in resource.list(): - if lb.name.startswith(f"kube_service_{cluster}_"): - yield lb - - -async def secgroups_for_cluster(resource, cluster): - """Async iterator for security groups belonging to the specified cluster.""" - async for sg in resource.list(): - if not sg.description.startswith("Security Group for"): - continue - if not sg.description.endswith(f"Service LoadBalancer in cluster {cluster}"): - continue - yield sg - - -async def filtered_volumes_for_cluster(resource, cluster): - """Async iterator for volumes belonging to the specified cluster.""" - async for vol in resource.list(): - # CSI Cinder sets metadata on the volumes that we can look for - owner = vol.metadata.get("cinder.csi.openstack.org/cluster") - # Skip volumes with the keep property set to true - if ( - owner - and owner == cluster - and vol.metadata.get(OPENSTACK_USER_VOLUMES_RECLAIM_PROPERTY) != "true" - ): - yield vol - - -async def snapshots_for_cluster(resource, cluster): - """Async iterator for snapshots belonging to the specified cluster.""" - async for snapshot in resource.list(): - # CSI Cinder sets metadata on the volumes that we can look for - owner = snapshot.metadata.get("cinder.csi.openstack.org/cluster") - if owner and owner == cluster: - yield snapshot - - -async def empty(async_iterator): - """Returns True if the given async iterator is empty, False otherwise.""" - try: - _ = await async_iterator.__anext__() - except StopAsyncIteration: - return True - else: - return False - - -async def try_delete(logger, resource, instances, **kwargs): - """Tries to delete the specified instances, catches 400 & 409 exceptions for retry. - - It returns a boolean indicating whether a check is required for the resource. - """ - check_required = False - async for instance in instances: - check_required = True - try: - await resource.delete(instance.id, **kwargs) - except httpx.HTTPStatusError as exc: - if exc.response.status_code in {400, 409}: - logger.warn( - f"got status code {exc.response.status_code} when trying to delete " - f"{resource.singular_name} with ID {instance.id} - will retry" - ) - else: - raise - return check_required - - -async def purge_openstack_resources( # noqa: C901 - logger, - clouds, - cloud_name, - cacert, - name, - include_volumes, - include_loadbalancers, - include_appcred, -): - """Cleans up the OpenStack resources created by the OCCM and CSI for a cluster.""" - # Use the credential to delete external resources as required - async with openstack.Cloud.from_clouds(clouds, cloud_name, cacert) as cloud: - if not cloud.is_authenticated: - if include_appcred: - # If the session is not authenticated then we've already - # cleaned up and deleted the app cred. - logger.warn("application credential has been deleted") - else: - # Raise an error and skip removing the finalizer to block cluster - # deletion to avoid leaking resources. - raise openstack.AuthenticationError(cloud.current_user_id) - return - - # Release any floating IPs associated with loadbalancer services for the cluster - networkapi = cloud.api_client("network", "/v2.0/") - fips = networkapi.resource("floatingips") - check_fips = await try_delete(logger, fips, fips_for_cluster(fips, name)) - logger.info("deleted floating IPs for LoadBalancer services") - - # Delete any loadbalancers associated with loadbalancer services for the cluster - lbapi = cloud.api_client("load-balancer", "/v2/lbaas/") - loadbalancers = lbapi.resource("loadbalancers") - check_lbs = False - if include_loadbalancers: - check_lbs = await try_delete( - logger, - loadbalancers, - lbs_for_cluster(loadbalancers, name), - cascade="true", - ) - logger.info("deleted load balancers for LoadBalancer services") - - # Delete security groups associated with loadbalancer services for the cluster - secgroups = networkapi.resource("security-groups") - check_secgroups = await try_delete( - logger, secgroups, secgroups_for_cluster(secgroups, name) - ) - logger.info("deleted security groups for LoadBalancer services") - - # Delete volumes and snapshots associated with PVCs, unless requested - # otherwise via the annotation - # NOTE(sd109): DevStack uses 'block-storage' as the Cinder service - # type in the catalog which is technically correct and volumev3 is just - # a historically valid alias, see: - # - https://docs.openstack.org/keystone/latest/contributor/service-catalog.html - # - https://service-types.openstack.org/service-types.json - # TODO(sd109): Make use of https://opendev.org/openstack/os-service-types to - # improve service type alias handling? - try: - volumeapi = cloud.api_client("volumev3") - except KeyError: - try: - volumeapi = cloud.api_client("block-storage") - except KeyError: - raise openstack.CatalogError("volumev3 or block-storage") - - snapshots_detail = volumeapi.resource("snapshots/detail") - snapshots = volumeapi.resource("snapshots") - check_snapshots = False - volumes_detail = volumeapi.resource("volumes/detail") - volumes = volumeapi.resource("volumes") - check_volumes = False - if include_volumes: - check_snapshots = await try_delete( - logger, snapshots, snapshots_for_cluster(snapshots_detail, name) - ) - logger.info("deleted snapshots for persistent volume claims") - check_volumes = await try_delete( - logger, volumes, filtered_volumes_for_cluster(volumes_detail, name) - ) - logger.info("deleted volumes for persistent volume claims") - - # Check that the resources have actually been deleted - if check_fips and not await empty(fips_for_cluster(fips, name)): - raise ResourcesStillPresentError("floatingips", name) - if check_lbs and not await empty(lbs_for_cluster(loadbalancers, name)): - raise ResourcesStillPresentError("loadbalancers", name) - if check_secgroups and not await empty(secgroups_for_cluster(secgroups, name)): - raise ResourcesStillPresentError("security-groups", name) - if check_volumes and not await empty( - filtered_volumes_for_cluster(volumes_detail, name) - ): - raise ResourcesStillPresentError("volumes", name) - if check_snapshots and not await empty( - snapshots_for_cluster(snapshots_detail, name) - ): - raise ResourcesStillPresentError("snapshots", name) - - # Now we have finished deleting resources, try to delete the appcred itself - # This requires an appcred to be unrestricted - # If it is not, we proceed but emit a warning - if include_appcred: - identityapi = cloud.api_client("identity", "v3") - appcreds = identityapi.resource( - "application_credentials", - # appcreds are user-namespaced - prefix=f"users/{cloud.current_user_id}", - ) - appcred_id = clouds["clouds"]["openstack"]["auth"][ - "application_credential_id" - ] - try: - await appcreds.delete(appcred_id) - except httpx.HTTPStatusError as exc: - if exc.response.status_code == 403: - logger.warn("unable to delete application credential for cluster") - else: - raise - logger.info("deleted application credential for cluster") - - -async def patch_finalizers(resource, name, namespace, finalizers): - """Patches the finalizers of a resource. - - If the resource does not exist anymore, that is classed as a success. - """ - try: - await resource.patch( - name, {"metadata": {"finalizers": finalizers}}, namespace=namespace - ) - except easykube.ApiError as exc: - # Patching the finalizers can result in a 422 if we are deleting and CAPO - # has removed its finalizer while we were working - if exc.status_code == 422: - raise kopf.TemporaryError("error patching finalizers", delay=1) - elif exc.status_code != 404: - raise - - -def retry_event(handler): - """Decorator for retrying events on Kubernetes objects. - - Instead of retrying within the handler, potentially on stale data, the object is - annotated with the number of times it has been retried. This triggers a new event - for the retry which contains up-to-date data. - - As recommended in the kopf docs, handlers should be idempotent as this mechanism - may result in the handler being called twice if an update happens between a failure - and the associated retry. - """ - - @functools.wraps(handler) - async def wrapper(**kwargs): - body = kwargs["body"] - resource = await ekclient.api(body["apiVersion"]).resource(body["kind"]) - try: - return await handler(**kwargs) - except Exception as exc: - if isinstance(exc, kopf.TemporaryError): - kwargs["logger"].warn(str(exc)) - backoff = exc.delay - elif isinstance( - exc, FinalizerStillPresentError | ResourcesStillPresentError - ): - kwargs["logger"].warn(str(exc)) - backoff = 5 - else: - kwargs["logger"].exception(str(exc)) - # Calculate the backoff - backoff = RETRY_DEFAULT_DELAY - # Wait for the backoff before annotating the resource - if backoff is None: - backoff = RETRY_DEFAULT_DELAY - await asyncio.sleep(backoff) - # Annotate the object with a random value to trigger another event - try: - await resource.patch( - kwargs["name"], - { - "metadata": { - "annotations": { - RETRY_ANNOTATION: "".join( - random.choices( - string.ascii_lowercase + string.digits, k=8 - ) - ), - } - } - }, - namespace=kwargs["namespace"], - ) - except easykube.ApiError as exc: - if exc.status_code != 404: - raise - - return wrapper - - -@kopf.on.event(CAPO_API_GROUP, "openstackclusters") -@retry_event -async def on_openstackcluster_event( - name, namespace, meta, labels, spec, status, logger, **kwargs -): - await _on_openstackcluster_event_impl( - name, namespace, meta, labels, spec, status, logger, **kwargs - ) - - -async def _on_openstackcluster_event_impl( - name, namespace, meta, labels, spec, status, logger, **kwargs -): - """Executes whenever an event occurs for an OpenStack cluster.""" - # Get the resource for manipulating OpenStackClusters at the preferred version - openstackclusters = await _get_os_cluster_client() - - # Use the value of the `cluster.x-k8s.io/cluster-name` label as the cluster name - # if it exists, otherwise, fall back to the OpenStackCluster resource name. - clustername = labels.get("cluster.x-k8s.io/cluster-name", name) - logger.debug(f"cluster name that will be used for cleanup: '{clustername}'") - - finalizers = meta.get("finalizers", []) - # We add a custom finalizer to OpenStack cluster objects to - # prevent them from being deleted until we have acted - if not meta.get("deletionTimestamp"): - if FINALIZER not in finalizers: - await patch_finalizers( - openstackclusters, - name, - namespace, - [*finalizers, FINALIZER], - ) - logger.info("added janitor finalizer to cluster") - return - - # NOTE: If we get to here, the cluster is deleting - - # If our finalizer is not present, we don't do anything - if FINALIZER not in finalizers: - logger.info("janitor finalizer not present, skipping cleanup") - return - - # Get the cloud credential from the cluster and use it to delete dangling - # resources created by OpenStack integrations on the cluster - clouds_secret = await _get_clouds_secret( - spec["identityRef"]["name"], namespace=namespace - ) - if clouds_secret is None: - # TODO(johngarbutt): fail better when secret not found? - logger.error(f"clouds.yaml not found for: {clustername}") - - else: - clouds = yaml.safe_load(base64.b64decode(clouds_secret.data["clouds.yaml"])) - if "cacert" in clouds_secret.data: - cacert = base64.b64decode(clouds_secret.data["cacert"]).decode() - else: - cacert = None - # The cloud name comes from spec.identityRef.cloudName from v1beta1 onwards - # Prior to that, it comes from spec.cloudName - cloud_name = spec["identityRef"].get( - "cloudName", spec.get("cloudName", "openstack") - ) - # The value of this annotation on the cluster decides whether to delete volumes - volumes_annotation_value = meta.get("annotations", {}).get( - VOLUMES_ANNOTATION, VOLUMES_ANNOTATION_DEFAULT - ) - # We want to remove the appcred iff: - # - # 1. The annotation on the secret is present and says delete - # 2. Our finalizer is the last finalizer - # - # This is because we need to allow CAPO to finish its work before we remove the - # appcred + secret, but we do need to act to remove other resources that might - # block CAPO from completing - credential_annotation_value = clouds_secret.metadata.get("annotations", {}).get( - CREDENTIAL_ANNOTATION - ) - remove_appcred = credential_annotation_value == CREDENTIAL_ANNOTATION_DELETE - - # Handle the case where the API server load balancer is enabled but was - # never successfully created (possibly due to LB permissions error) - # meaning that there cannot be any other LBs to remove - # (since API server was never online due to missing load balancer) - remove_loadbalancers = ( - # Default to false since this is default CAPO behaviour if - # ApiServerLoadBalancer field is omitted from OpenStackClusterSpec - # https://github.com/kubernetes-sigs/cluster-api-provider-openstack/blob/57ae27ee114bda3606d92163397697b640272673/api/v1beta1/openstackcluster_types.go#L99-L101 - spec.get("apiServerLoadBalancer", {}).get("enabled", False) - and status.get("apiServerLoadBalancer", {}).get("id", "") != "" - ) - - await purge_openstack_resources( - logger, - clouds, - cloud_name, - cacert, - clustername, - volumes_annotation_value == VOLUMES_ANNOTATION_DELETE, - remove_loadbalancers, - remove_appcred and len(finalizers) == 1, - ) - - # If we get to here, OpenStack resources have been successfully deleted - # So we can remove the appcred secret if we are the last actor - if remove_appcred and len(finalizers) == 1: - await _delete_secret(clouds_secret.metadata["name"], namespace) - logger.info("cloud credential secret deleted") - elif remove_appcred: - # If the annotation says delete but other controllers are still acting, - # go round again - raise FinalizerStillPresentError( - next(f for f in finalizers if f != FINALIZER), name - ) - - # If we get to here, we can remove the finalizer - await patch_finalizers( - openstackclusters, name, namespace, [f for f in finalizers if f != FINALIZER] - ) - logger.info("removed janitor finalizer from cluster") - - -async def _delete_secret(name, namespace): - secrets = await ekclient.api("v1").resource("secrets") - await secrets.delete(name, namespace=namespace) - - -async def _get_os_cluster_client(): - capoapi = await ekclient.api_preferred_version(CAPO_API_GROUP) - openstackclusters = await capoapi.resource("openstackclusters") - return openstackclusters - - -async def _get_clouds_secret(secret_name, namespace): - secrets = await ekclient.api("v1").resource("secrets") - try: - return await secrets.fetch(secret_name, namespace=namespace) - except easykube.ApiError as exc: - if exc.status_code != 404: - raise - # TODO(johngarbutt): fail better when not found? diff --git a/capi_janitor/tests/__init__.py b/capi_janitor/tests/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/capi_janitor/tests/openstack/__init__.py b/capi_janitor/tests/openstack/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/capi_janitor/tests/openstack/test_openstack.py b/capi_janitor/tests/openstack/test_openstack.py deleted file mode 100644 index 444303c..0000000 --- a/capi_janitor/tests/openstack/test_openstack.py +++ /dev/null @@ -1,280 +0,0 @@ -import unittest -from unittest.mock import AsyncMock, MagicMock, patch - -from capi_janitor.openstack.openstack import AuthenticationError, Cloud - - -class TestCloudAsyncContext(unittest.IsolatedAsyncioTestCase): - # Set up common variables for all tests - async def asyncSetUp(self): - # Auth is mocked to simulate authentication - self.auth = MagicMock() - self.auth.url = "https://example.com:5000/identity" - # Transport is awaited so can be Async Mocked - self.transport = AsyncMock() - # Interface & Region can be fixed for the tests - self.interface = "public" - self.region = "region1" - # Create a Cloud instance with the mocked auth and transport - self.cloud = Cloud(self.auth, self.transport, self.interface, self.region) - - # Test the __aenter__ method for auth success and general functionality - @patch("capi_janitor.openstack.openstack.Client") - async def test_cloud_successful_authentication(self, mock_client): - # Patched client to simulate a successful authentication - mock_client_instance = AsyncMock() - # Return mock for the client - mock_client.return_value = mock_client_instance - # Mock the get method to return a simple successful response - mock_client_instance.get.return_value.json = MagicMock( - return_value={ - "catalog": [ - { - "type": "compute", - "endpoints": [ - { - "interface": "public", - "region_id": "region1", - "url": "https://compute.example.com", - } - ], - } - ] - } - ) - # Mock the base_url for the client - mock_client_instance._base_url = "https://compute.example.com" - - # Assert return values - async with self.cloud as cloud: - self.assertTrue(cloud.is_authenticated) - self.assertIn("compute", cloud.apis) - self.assertEqual( - cloud.api_client("compute")._base_url, "https://compute.example.com" - ) - # Full URL must be used so the /identity path prefix is preserved. - # Client.__init__ strips the path into _prefix (not used by raw - # client.get()), so a relative "/v3/auth/catalog" would resolve - # against base_url "https://example.com:5000" and lose /identity. - mock_client_instance.get.assert_called_once_with( - "https://example.com:5000/identity/v3/auth/catalog" - ) - - # Test the __aenter__ method for auth failure - @patch("capi_janitor.openstack.openstack.Client") - async def test_cloud_authentication_failure(self, mock_client): - mock_client_instance = AsyncMock() - mock_client.return_value = mock_client_instance - # Simulate an auth error with a named user - mock_client_instance.get.side_effect = AuthenticationError("test_user") - - with self.assertRaises(AuthenticationError) as context: - async with self.cloud: - pass - # Assert that the AuthenticationError is raised with the correct message - self.assertEqual( - str(context.exception), "failed to authenticate as user: test_user" - ) - - # Test the __aenter__ method for 404 error - @patch("capi_janitor.openstack.openstack.Client") - async def test_cloud_auth_404_error(self, mock_client): - mock_client_instance = AsyncMock() - mock_client.return_value = mock_client_instance - # Simulate a 404 error - mock_client_instance.get.side_effect = MagicMock( - response=MagicMock(status_code=404) - ) - - # Assert auth failed and no endpoints are returned - async with self.cloud as cloud: - self.assertFalse(cloud.is_authenticated) - self.assertEqual(cloud.apis, []) - - # Test the __aenter__ method for no matching interface - @patch("capi_janitor.openstack.openstack.Client") - async def test_cloud_no_matching_interface(self, mock_client): - mock_client_instance = AsyncMock() - mock_client.return_value = mock_client_instance - # No matching interface in the response - mock_client_instance.get.return_value.json = MagicMock( - return_value={ - "catalog": [ - { - "type": "compute", - "endpoints": [ - { - "interface": "internal", - "region_id": "region1", - "url": "https://compute.example.com", - } - ], - } - ] - } - ) - - async with self.cloud as cloud: - self.assertFalse(cloud.is_authenticated) - self.assertEqual(cloud.apis, []) - - @patch("capi_janitor.openstack.openstack.Client") - async def test_cloud_no_matching_region_id(self, mock_client): - mock_client_instance = AsyncMock() - mock_client.return_value = mock_client_instance - # No matching region_id in the response - mock_client_instance.get.return_value.json = MagicMock( - return_value={ - "catalog": [ - { - "type": "compute", - "endpoints": [ - { - "interface": "public", - "region_id": "region2", - "url": "https://compute.example.com", - } - ], - } - ] - } - ) - - async with self.cloud as cloud: - self.assertFalse(cloud.is_authenticated) - self.assertEqual(cloud.apis, []) - - @patch("capi_janitor.openstack.openstack.Client") - async def test_cloud_filter_endpoints(self, mock_client): - mock_client_instance = AsyncMock() - mock_client.return_value = mock_client_instance - # Return multiple endpoints, one matching, one not - mock_client_instance.get.return_value.json = MagicMock( - return_value={ - "catalog": [ - { - "type": "compute", - "endpoints": [ - { - "interface": "public", - "region_id": "region1", - "url": "https://compute.example.com", - } - ], - }, - { - "type": "network", - "endpoints": [ - { - "interface": "internal", - "region_id": "region1", - "url": "https://network.example.com", - } - ], - }, - ] - } - ) - - async with self.cloud as cloud: - self.assertTrue(cloud.is_authenticated) - self.assertIn("compute", cloud.apis) - self.assertNotIn("network", cloud.apis) - - @patch("capi_janitor.openstack.openstack.Client") - async def test_cloud_multiple_services(self, mock_client): - mock_client_instance = AsyncMock() - mock_client.return_value = mock_client_instance - # Return multiple services, some matching, some not - mock_client_instance.get.return_value.json = MagicMock( - return_value={ - "catalog": [ - { - "type": "compute", - "endpoints": [ - { - "interface": "public", - "region_id": "region1", - "url": "https://compute.example.com", - } - ], - }, - { - "type": "storage", - "endpoints": [ - { - "interface": "internal", - "region_id": "region1", - "url": "https://storage.example.com", - } - ], - }, - { - "type": "network", - "endpoints": [ - { - "interface": "public", - "region_id": "region1", - "url": "https://network.example.com", - } - ], - }, - ] - } - ) - - async with self.cloud as cloud: - self.assertTrue(cloud.is_authenticated) - self.assertIn("compute", cloud.apis) - self.assertNotIn("storage", cloud.apis) - self.assertIn("network", cloud.apis) - - @patch("capi_janitor.openstack.openstack.Client") - async def test_cloud_empty_endpoint_list(self, mock_client): - mock_client_instance = AsyncMock() - mock_client.return_value = mock_client_instance - mock_client_instance.get.return_value.json = MagicMock( - return_value={"catalog": [{"type": "compute", "endpoints": []}]} - ) - - async with self.cloud as cloud: - self.assertFalse(cloud.is_authenticated) - - @patch("capi_janitor.openstack.openstack.Client") - async def test_cloud_no_region_specified(self, mock_client): - # Set up the cloud instance without a region - self.cloud = Cloud(self.auth, self.transport, self.interface, region=None) - mock_client_instance = AsyncMock() - mock_client.return_value = mock_client_instance - # Return endpoints with different region_ids - mock_client_instance.get.return_value.json = MagicMock( - return_value={ - "catalog": [ - { - "type": "compute", - "endpoints": [ - { - "interface": "public", - "region_id": "region1", - "url": "https://compute.example.com", - } - ], - }, - { - "type": "network", - "endpoints": [ - { - "interface": "public", - "region_id": "region2", - "url": "https://network.example.com", - } - ], - }, - ] - } - ) - - async with self.cloud as cloud: - self.assertTrue(cloud.is_authenticated) - self.assertIn("compute", cloud.apis) - self.assertIn("network", cloud.apis) diff --git a/capi_janitor/tests/openstack/test_operator.py b/capi_janitor/tests/openstack/test_operator.py deleted file mode 100644 index 5792d8d..0000000 --- a/capi_janitor/tests/openstack/test_operator.py +++ /dev/null @@ -1,475 +0,0 @@ -import base64 -import unittest -from unittest import mock - -import easykube -import httpx -import yaml - -from capi_janitor.openstack import openstack, operator -from capi_janitor.openstack.operator import OPENSTACK_USER_VOLUMES_RECLAIM_PROPERTY - - -# Helper to create an async iterable -class AsyncIterList: - def __init__(self, items): - self.items = items - self.kwargs = None - - async def list(self, **kwargs): - self.kwargs = kwargs - for item in self.items: - yield item - - def __aiter__(self): - self._iter = iter(self.items) - return self - - async def __anext__(self): - try: - return next(self._iter) - except StopIteration: - raise StopAsyncIteration - - -class TestOperator(unittest.IsolatedAsyncioTestCase): - async def test_operator(self): - mock_easykube = mock.AsyncMock(spec=easykube.AsyncClient) - operator.ekclient = mock_easykube - - await operator.on_cleanup() # type: ignore - - mock_easykube.aclose.assert_awaited_once_with() - - async def test_fips_for_cluster(self): - prefix = "Floating IP for Kubernetes external service from cluster" - fips = [ - mock.Mock(description=f"{prefix} mycluster"), - mock.Mock(description=f"{prefix} asdf"), - mock.Mock(description="Some other description"), - mock.Mock(description=f"{prefix} mycluster"), - ] - resource_mock = AsyncIterList(fips) - - result = [ - fip async for fip in operator.fips_for_cluster(resource_mock, "mycluster") - ] - - self.assertEqual(len(result), 2) - self.assertEqual(result[0], fips[0]) - self.assertEqual(result[1], fips[3]) - self.assertEqual(resource_mock.kwargs, {}) - - async def test_lbs_for_cluster(self): - lbs = [ - mock.Mock(name="lb0"), - mock.Mock(name="lb1"), - mock.Mock(name="lb2"), - mock.Mock(name="lb3"), - ] - # can't pass name into mock.Mock() to do this - lbs[0].name = "kube_service_mycluster_api" - lbs[1].name = "kube_service_othercluster_api" - lbs[2].name = "fake_service_mycluster_api" - lbs[3].name = "kube_service_mycluster_ui" - resource_mock = AsyncIterList(lbs) - - result = [ - lb async for lb in operator.lbs_for_cluster(resource_mock, "mycluster") - ] - - self.assertEqual(len(result), 2) - self.assertEqual(result[0], lbs[0]) - self.assertEqual(result[1], lbs[3]) - - async def test_secgroups_for_cluster(self): - prefix = "Security Group for Service LoadBalancer in cluster" - secgroups = [ - mock.Mock(description=f"{prefix} mycluster"), - mock.Mock(description=f"{prefix} othercluster"), - mock.Mock(description="Other description"), - mock.Mock(description=f"{prefix} mycluster"), - ] - resource_mock = AsyncIterList(secgroups) - - result = [] - async for sg in operator.secgroups_for_cluster(resource_mock, "mycluster"): - result.append(sg) - - self.assertEqual(len(result), 2) - self.assertEqual(result[0], secgroups[0]) - self.assertEqual(result[1], secgroups[3]) - - async def test_filtered_volumes_for_cluster(self): - volumes = [ - mock.Mock( - metadata={ - "cinder.csi.openstack.org/cluster": "mycluster", - OPENSTACK_USER_VOLUMES_RECLAIM_PROPERTY: "false", - } - ), - mock.Mock( - metadata={ - "cinder.csi.openstack.org/cluster": "mycluster", - OPENSTACK_USER_VOLUMES_RECLAIM_PROPERTY: "true", - } - ), - mock.Mock( - metadata={ - "cinder.csi.openstack.org/cluster": "othercluster", - OPENSTACK_USER_VOLUMES_RECLAIM_PROPERTY: "false", - } - ), - mock.Mock( - metadata={ - "cinder.csi.openstack.org/cluster": "mycluster", - } - ), - mock.Mock(metadata={"other_key": "value"}), - ] - resource_mock = AsyncIterList(volumes) - - result = [] - async for vol in operator.filtered_volumes_for_cluster( - resource_mock, "mycluster" - ): - result.append(vol) - - self.assertEqual(len(result), 2) - self.assertEqual(result[0], volumes[0]) - self.assertEqual(result[1], volumes[3]) - - async def test_snapshots_for_cluster(self): - snapshots = [ - mock.Mock(metadata={"cinder.csi.openstack.org/cluster": "mycluster"}), - mock.Mock(metadata={"cinder.csi.openstack.org/cluster": "othercluster"}), - mock.Mock(metadata={"cinder.csi.openstack.org/cluster": "mycluster"}), - mock.Mock(metadata={"cinder.csi.openstack.org/cluster": "othercluster"}), - # Volumes with invalid metadata - mock.Mock(metadata={"another_key": "value"}), - ] - resource_mock = AsyncIterList(snapshots) - - result = [] - async for snap in operator.snapshots_for_cluster(resource_mock, "mycluster"): - result.append(snap) - - self.assertEqual(len(result), 2) - self.assertEqual(result[0], snapshots[0]) - self.assertEqual(result[1], snapshots[2]) - - async def test_empty_iterator_returns_true(self): - async_iter = AsyncIterList([]).__aiter__() - result = await operator.empty(async_iter) - self.assertTrue(result) - - async def test_non_empty_iterator_returns_false(self): - async_iter = AsyncIterList([1, 2, 3]).__aiter__() - result = await operator.empty(async_iter) - self.assertFalse(result) - - async def test_try_delete_success(self): - instances = [mock.Mock(id=1), mock.Mock(id=2)] - resource = mock.Mock() - resource.delete = mock.AsyncMock() - resource.singular_name = "test_resource" - logger = mock.Mock() - - result = await operator.try_delete(logger, resource, AsyncIterList(instances)) - - self.assertTrue(result) - resource.delete.assert_any_await(1) - resource.delete.assert_any_await(2) - logger.warn.assert_not_called() - - async def test_try_delete_with_400_and_409_errors(self): - instances = [mock.Mock(id=1), mock.Mock(id=2)] - resource = mock.Mock() - resource.singular_name = "test_resource" - logger = mock.Mock() - - resource.delete = mock.AsyncMock( - side_effect=[ - httpx.HTTPStatusError( - "error", request=mock.Mock(), response=mock.Mock(status_code=400) - ), - httpx.HTTPStatusError( - "error", request=mock.Mock(), response=mock.Mock(status_code=409) - ), - ] - ) - result = await operator.try_delete(logger, resource, AsyncIterList(instances)) - - self.assertTrue(result) - self.assertEqual(resource.delete.await_count, 2) - self.assertEqual(logger.warn.call_count, 2) - - async def test_delete_with_unexpected_error_raises(self): - instances = [mock.Mock(id=1)] - resource = mock.Mock() - resource.singular_name = "test_resource" - logger = mock.Mock() - - resource.delete = mock.AsyncMock( - side_effect=httpx.HTTPStatusError( - "error", request=mock.Mock(), response=mock.Mock(status_code=500) - ) - ) - with self.assertRaises(httpx.HTTPStatusError): - await operator.try_delete(logger, resource, AsyncIterList(instances)) - - @mock.patch.object(operator, "patch_finalizers") - @mock.patch.object(operator, "_get_os_cluster_client") - async def test_on_openstackcluster_event_adds_finalizers( - self, mock_get_client, mock_patch_finalizers - ): - logger = mock.Mock() - mock_get_client.return_value = "mock_client" - - await operator._on_openstackcluster_event_impl( - name="mycluster", - namespace="default", - meta={}, - labels={}, - spec={}, - status={}, - logger=logger, - body={}, - ) - - mock_patch_finalizers.assert_awaited_once_with( - "mock_client", "mycluster", "default", ["janitor.capi.stackhpc.com"] - ) - logger.debug.assert_has_calls( - [mock.call("cluster name that will be used for cleanup: 'mycluster'")] - ) - logger.info.assert_has_calls([mock.call("added janitor finalizer to cluster")]) - - @mock.patch.object(operator, "patch_finalizers") - @mock.patch.object(operator, "_get_os_cluster_client") - async def test_on_openstackcluster_event_skip_no_finalizers( - self, mock_get_client, mock_patch_finalizers - ): - logger = mock.Mock() - mock_get_client.return_value = "mock_client" - - await operator._on_openstackcluster_event_impl( - name="mycluster", - namespace="default", - meta={"deletionTimestamp": "2023-10-01T00:00:00Z"}, - labels={}, - spec={}, - status={}, - logger=logger, - body={}, - ) - - mock_patch_finalizers.assert_not_awaited() - logger.debug.assert_has_calls( - [mock.call("cluster name that will be used for cleanup: 'mycluster'")] - ) - logger.info.assert_has_calls( - [mock.call("janitor finalizer not present, skipping cleanup")] - ) - - @mock.patch.object(operator, "_delete_secret") - @mock.patch.object(operator, "_get_clouds_secret") - @mock.patch.object(operator, "purge_openstack_resources") - @mock.patch.object(operator, "patch_finalizers") - @mock.patch.object(operator, "_get_os_cluster_client") - async def test_on_openstackcluster_event_calls_purge( - self, - mock_get_client, - mock_patch_finalizers, - mock_purge, - mock_clouds_secret, - mock_delete_secret, - ): - logger = mock.Mock() - mock_get_client.return_value = "mock_client" - mock_secret = mock.Mock() - mock_clouds_secret.return_value = mock_secret - clouds_yaml_data = { - "openstack": { - "auth": { - "auth_url": "https://example.com:5000/v3", - "username": "user", - "password": "pass", - "project_name": "project", - "user_domain_name": "Default", - "project_domain_name": "Default", - } - } - } - mock_secret.data = { - "clouds.yaml": base64.b64encode(yaml.dump(clouds_yaml_data).encode("utf-8")) - } - mock_secret.metadata = { - "annotations": {"janitor.capi.stackhpc.com/credential-policy": "delete"}, - "name": "appcred42", - } - - await operator._on_openstackcluster_event_impl( - name="mycluster", - namespace="namespace1", - meta={ - "deletionTimestamp": "2023-10-01T00:00:00Z", - "finalizers": ["janitor.capi.stackhpc.com"], - "annotations": { - "janitor.capi.stackhpc.com/volumes-policy": "delete", - }, - }, - labels={}, - spec={"identityRef": {"name": "appcred42"}}, - status={}, - logger=logger, - body={}, - ) - - mock_purge.assert_awaited_once_with( - logger, - clouds_yaml_data, - "openstack", - None, - "mycluster", - True, - False, - True, - ) - mock_delete_secret.assert_awaited_once_with("appcred42", "namespace1") - mock_patch_finalizers.assert_awaited_once_with( - "mock_client", "mycluster", "namespace1", [] - ) - logger.debug.assert_has_calls( - [mock.call("cluster name that will be used for cleanup: 'mycluster'")] - ) - logger.info.assert_has_calls( - [ - mock.call("cloud credential secret deleted"), - mock.call("removed janitor finalizer from cluster"), - ] - ) - - @mock.patch.object(openstack.Cloud, "from_clouds") - async def test_purge_openstack_resources_raises(self, mock_from_clouds): - mock_networkapi = mock.AsyncMock() - mock_networkapi.resource.side_effect = lambda resource: resource - - mock_cloud = mock.AsyncMock() - mock_cloud.__aenter__.return_value = mock_cloud - mock_cloud.is_authenticated = False - mock_cloud.current_user_id = "user" - mock_cloud.api_client.return_value = mock_networkapi - - mock_from_clouds.return_value = mock_cloud - - logger = mock.Mock() - clouds_yaml_data = { - "clouds": { - "openstack": { - "auth": { - "auth_url": "https://example.com:5000/v3", - "application_credential_id": "user", - "application_credential_secret": "pass", - }, - "region_name": "RegionOne", - "interface": "public", - "identity_api_version": 3, - "auth_type": "v3applicationcredential", - } - } - } - with self.assertRaises(openstack.AuthenticationError) as e: - await operator.purge_openstack_resources( - logger, - clouds_yaml_data, - "openstack", - None, - "mycluster", - True, - True, - False, - ) - self.assertEqual( - str(e.exception), - "failed to authenticate as user: user", - ) - - # @mock.patch.object(openstack.Cloud, "from_clouds") - # async def test_purge_openstack_resources_success(self, mock_from_clouds): - # # Mocking the cloud object and API clients - # mock_cloud = mock.AsyncMock() - # mock_networkapi = mock.AsyncMock() - # mock_lbapi = mock.AsyncMock() - # mock_volumeapi = mock.AsyncMock() - # mock_identityapi = mock.AsyncMock() - - # # Mock the __aenter__ to return the mock_cloud - # mock_cloud.__aenter__.return_value = mock_cloud - # mock_cloud.is_authenticated = True - # mock_cloud.current_user_id = "user" - # mock_cloud.api_client.return_value = mock_networkapi - - # # Return mock clients for different services when requested - # mock_from_clouds.return_value = mock_cloud - - # # Mocking the resources for Network, Load Balancer, and Volume APIs - # mock_networkapi.resource.side_effect = lambda resource: { - # "floatingips": mock.AsyncMock(), - # "security-groups": mock.AsyncMock(), - # }.get(resource, None) - - # mock_lbapi.resource.side_effect = lambda resource: { - # "loadbalancers": mock.AsyncMock(), - # }.get(resource, None) - - # mock_volumeapi.resource.side_effect = lambda resource: { - # "snapshots/detail": mock.AsyncMock(), - # "snapshots": mock.AsyncMock(), - # "volumes/detail": mock.AsyncMock(), - # "volumes": mock.AsyncMock(), - # }.get(resource, None) - - # mock_identityapi.resource.side_effect = lambda resource: { - # "application_credentials": mock.AsyncMock(), - # }.get(resource, None) - - # # Mock logger - # logger = mock.Mock() - - # clouds_yaml_data = { - # "clouds": { - # "openstack": { - # "auth": { - # "auth_url": "https://example.com:5000/v3", - # "application_credential_id": "user", - # "application_credential_secret": "pass", - # }, - # "region_name": "RegionOne", - # "interface": "public", - # "identity_api_version": 3, - # "auth_type": "v3applicationcredential", - # } - # } - # } - - # # Simulate the purge_openstack_resources method behavior - # await operator.purge_openstack_resources( - # logger, - # clouds_yaml_data, # Pass the mock cloud config - # "openstack", - # None, - # "mycluster", - # True, - # False, - # ) - - # # Add assertions here based on expected behavior - # # Example: Check that the resources were interacted with - # mock_networkapi.resource.assert_any_call("floatingips") - # mock_lbapi.resource.assert_any_call("loadbalancers") - # mock_volumeapi.resource.assert_any_call("snapshots") - # mock_volumeapi.resource.assert_any_call("volumes") - - # # Example: Validate if appcred deletion was attempted - # mock_identityapi.resource.assert_any_call("application_credentials") diff --git a/pyproject.toml b/pyproject.toml deleted file mode 100644 index b68657a..0000000 --- a/pyproject.toml +++ /dev/null @@ -1,44 +0,0 @@ -[tool.ruff] -line-length = 88 -target-version = "py312" - -[tool.ruff.format] -line-ending = "lf" - -[tool.ruff.lint] -select = [ - # pycodestyle - "E", - # Pyflakes - "F", - # flake8-builtins - "A", - # Ruff rules - "RUF", - # flake8-async - "ASYNC", - # pyupgrade - "UP", - # tidy imports - "TID", - # sorted imports - "I", - # check complexity - "C90", - # pep8 naming - "N", -] - -ignore = [ - "UP017", # remove and fix once 3.10 not tested in CI -] - -[tool.mypy] -follow_imports = "silent" -warn_redundant_casts = true -warn_unused_ignores = true -check_untyped_defs = true - -[tool.ruff.lint.mccabe] -# Flag errors (`C901`) whenever the complexity level exceeds 5. -max-complexity = 14 diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 868cbe1..0000000 --- a/requirements.txt +++ /dev/null @@ -1,24 +0,0 @@ -aiohappyeyeballs==2.7.1 -aiohttp==3.14.1 -aiosignal==1.4.0 -anyio==4.14.1 -async-timeout==5.0.1 -attrs==26.1.0 -certifi==2026.6.17 -click==8.4.2 -easykube==0.6.0 -exceptiongroup==1.3.1 -frozenlist==1.8.0 -h11==0.16.0 -httpcore==1.0.9 -httpx==0.28.1 -idna==3.18 -iso8601==2.1.0 -kopf==1.44.6 -multidict==6.7.1 -propcache==0.5.2 -python-json-logger==4.1.0 -PyYAML==6.0.3 -sniffio==1.3.1 -typing_extensions==4.16.0 -yarl==1.24.2 diff --git a/setup.cfg b/setup.cfg deleted file mode 100755 index 17d5b3f..0000000 --- a/setup.cfg +++ /dev/null @@ -1,18 +0,0 @@ -[metadata] -name = cluster-api-janitor-openstack -version = 0.1.0 -description = Kubernetes operator that cleans up external resources for Cluster API OpenStack clusters. -author = Matt Pryor -author_email = matt@stackhpc.com -url = https://github.com/azimuth-cloud/cluster-api-janitor-openstack -python-requires = ">=3.10" - -[options] -zip_safe = False -include_package_data = True -packages = find_namespace: -install_requires = - easykube - httpx - kopf - pyyaml diff --git a/setup.py b/setup.py deleted file mode 100755 index bac24a4..0000000 --- a/setup.py +++ /dev/null @@ -1,6 +0,0 @@ -#!/usr/bin/env python - -import setuptools - -if __name__ == "__main__": - setuptools.setup() diff --git a/tox.ini b/tox.ini deleted file mode 100644 index eaaa863..0000000 --- a/tox.ini +++ /dev/null @@ -1,55 +0,0 @@ -[tox] -minversion = 4.0.0 -# We run autofix last, to ensure CI fails, -# even though we do our best to autofix locally -envlist = py3,ruff,codespell,mypy,autofix -skipsdist = True - -[testenv] -basepython = python3 -usedevelop = True -setenv = - PYTHONWARNINGS=default::DeprecationWarning - OS_STDOUT_CAPTURE=1 - OS_STDERR_CAPTURE=1 - OS_TEST_TIMEOUT=60 -deps = -r{toxinidir}/requirements.txt - -r{toxinidir}/test-requirements.txt -commands = stestr run {posargs} - -[testenv:autofix] -commands = - ruff format {tox_root} - codespell {tox_root} -w --skip venv - ruff check {tox_root} --fix - -[testenv:black] -# TODO: understand why ruff doesn't fix -# line lengths as well as black does -commands = black {tox_root} --check - -[testenv:codespell] -commands = codespell --skip ./venv {posargs} - -[testenv:ruff] -description = Run Ruff checks -commands = - ruff check {tox_root} - ruff format {tox_root} --check - -[testenv:venv] -commands = {posargs} - -[testenv:cover] -setenv = - VIRTUAL_ENV={envdir} - PYTHON=coverage run --source capi_janitor --parallel-mode -commands = - stestr run {posargs} - coverage combine - coverage html -d cover - coverage xml -o cover/coverage.xml - coverage report - -[testenv:mypy] -commands = mypy {tox_root} {posargs} From 58996d9a7f938f08fc6034d70cf2b42c1db15a74 Mon Sep 17 00:00:00 2001 From: Mathieu Grzybek Date: Wed, 8 Jul 2026 14:12:20 +0200 Subject: [PATCH 18/27] fix(chart): fix packaging tests and update Helm snapshot MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - packaging_test.yaml: remove `hasDocuments: count: 4` assertion — helm-unittest applies it per template file (not globally), so each file fails because it only renders 1 document. The snapshot test already validates the full set of rendered resources. - snapshot_test.yaml.snap: regenerate to match current chart templates: - ClusterRole (snapshot 1): add "update" verb on openstackclusters - Deployment (snapshot 3): add CAPI_JANITOR_RETRY_DEFAULT_DELAY env var, livenessProbe (/healthz:8081) and readinessProbe (/readyz:8081) Signed-off-by: Mathieu Grzybek --- chart/tests/__snapshot__/snapshot_test.yaml.snap | 15 +++++++++++++++ chart/tests/packaging_test.yaml | 5 ----- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/chart/tests/__snapshot__/snapshot_test.yaml.snap b/chart/tests/__snapshot__/snapshot_test.yaml.snap index 59b69c4..2abd006 100644 --- a/chart/tests/__snapshot__/snapshot_test.yaml.snap +++ b/chart/tests/__snapshot__/snapshot_test.yaml.snap @@ -41,6 +41,7 @@ templated manifests should match snapshot: - get - watch - patch + - update - apiGroups: - apiextensions.k8s.io resources: @@ -97,9 +98,23 @@ templated manifests should match snapshot: - env: - name: CAPI_JANITOR_DEFAULT_VOLUMES_POLICY value: delete + - name: CAPI_JANITOR_RETRY_DEFAULT_DELAY + value: "60" image: ghcr.io/azimuth-cloud/cluster-api-janitor-openstack:main imagePullPolicy: IfNotPresent + livenessProbe: + httpGet: + path: /healthz + port: 8081 + initialDelaySeconds: 15 + periodSeconds: 20 name: operator + readinessProbe: + httpGet: + path: /readyz + port: 8081 + initialDelaySeconds: 5 + periodSeconds: 10 resources: {} securityContext: allowPrivilegeEscalation: false diff --git a/chart/tests/packaging_test.yaml b/chart/tests/packaging_test.yaml index 0614395..f67edc2 100644 --- a/chart/tests/packaging_test.yaml +++ b/chart/tests/packaging_test.yaml @@ -26,11 +26,6 @@ tests: content: ALL # US10.2 — Helm chart - - it: should create four resources (US10.2) - asserts: - - hasDocuments: - count: 4 - - it: should inject CAPI_JANITOR_DEFAULT_VOLUMES_POLICY with default value "delete" (US10.2) template: templates/deployment.yaml asserts: From 352eb64a930f4ddbf315e7fea237afb5c1ed71d3 Mon Sep 17 00:00:00 2001 From: Mathieu Grzybek Date: Wed, 8 Jul 2026 14:30:02 +0200 Subject: [PATCH 19/27] feat(ci): start from scratch using nix Signed-off-by: Mathieu Grzybek --- .custom-gcl.yml | 11 ------ .github/workflows/ci.yml | 23 ++++++++++++ .github/workflows/lint.yml | 29 -------------- .github/workflows/test-e2e.yml | 38 ------------------- .github/workflows/test.yml | 29 -------------- .golangci.yml | 69 ---------------------------------- Makefile | 24 ------------ nix/default.nix | 28 +++++++++++++- 8 files changed, 50 insertions(+), 201 deletions(-) delete mode 100644 .custom-gcl.yml create mode 100644 .github/workflows/ci.yml delete mode 100644 .github/workflows/lint.yml delete mode 100644 .github/workflows/test-e2e.yml delete mode 100644 .github/workflows/test.yml delete mode 100644 .golangci.yml diff --git a/.custom-gcl.yml b/.custom-gcl.yml deleted file mode 100644 index d9ae33b..0000000 --- a/.custom-gcl.yml +++ /dev/null @@ -1,11 +0,0 @@ -# This file configures golangci-lint with module plugins. -# When you run 'make lint', it will automatically build a custom golangci-lint binary -# with all the plugins listed below. -# -# See: https://golangci-lint.run/plugins/module-plugins/ -version: v2.12.2 -plugins: - # logcheck validates structured logging calls and parameters (e.g., balanced key-value pairs) - - module: "sigs.k8s.io/logtools" - import: "sigs.k8s.io/logtools/logcheck/gclplugin" - version: latest diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..4c44777 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,23 @@ +name: CI + +on: + push: + pull_request: + +permissions: {} + +jobs: + nix: + name: fmt + vet + test (Nix) + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - uses: DeterminateSystems/nix-installer-action@07ebb8d2749aa2fac2baae7d1cfa011e4acfd8d1 # v5 + + - name: Run fmt + vet + unit tests + run: nix-build nix -A tests diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml deleted file mode 100644 index d3f5b7b..0000000 --- a/.github/workflows/lint.yml +++ /dev/null @@ -1,29 +0,0 @@ -name: Lint - -on: - push: - pull_request: - -permissions: {} - -jobs: - lint: - permissions: - contents: read - name: Run on Ubuntu - runs-on: ubuntu-latest - steps: - - name: Clone the code - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - persist-credentials: false - - - name: Setup Go - uses: actions/setup-go@4b73464bb391d4059bd26b0524d20df3927bd417 # v6.3.0 - with: - go-version-file: go.mod - - - name: Check linter configuration - run: make lint-config - - name: Run linter - run: make lint diff --git a/.github/workflows/test-e2e.yml b/.github/workflows/test-e2e.yml deleted file mode 100644 index 83e295e..0000000 --- a/.github/workflows/test-e2e.yml +++ /dev/null @@ -1,38 +0,0 @@ -name: E2E Tests - -on: - push: - pull_request: - -permissions: {} - -jobs: - test-e2e: - permissions: - contents: read - name: Run on Ubuntu - runs-on: ubuntu-latest - steps: - - name: Clone the code - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - persist-credentials: false - - - name: Setup Go - uses: actions/setup-go@4b73464bb391d4059bd26b0524d20df3927bd417 # v6.3.0 - with: - go-version-file: go.mod - - - name: Install the latest version of kind - run: | - curl -Lo ./kind https://kind.sigs.k8s.io/dl/latest/kind-linux-$(go env GOARCH) - chmod +x ./kind - sudo mv ./kind /usr/local/bin/kind - - - name: Verify kind installation - run: kind version - - - name: Running Test e2e - run: | - go mod tidy - make test-e2e diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml deleted file mode 100644 index 27e82c8..0000000 --- a/.github/workflows/test.yml +++ /dev/null @@ -1,29 +0,0 @@ -name: Tests - -on: - push: - pull_request: - -permissions: {} - -jobs: - test: - permissions: - contents: read - name: Run on Ubuntu - runs-on: ubuntu-latest - steps: - - name: Clone the code - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - persist-credentials: false - - - name: Setup Go - uses: actions/setup-go@4b73464bb391d4059bd26b0524d20df3927bd417 # v6.3.0 - with: - go-version-file: go.mod - - - name: Running Tests - run: | - go mod tidy - make test diff --git a/.golangci.yml b/.golangci.yml deleted file mode 100644 index b139f79..0000000 --- a/.golangci.yml +++ /dev/null @@ -1,69 +0,0 @@ -version: "2" -run: - allow-parallel-runners: true -linters: - default: none - enable: - - copyloopvar - - depguard - - dupl - - errcheck - - ginkgolinter - - goconst - - gocyclo - - govet - - ineffassign - - lll - - modernize - - misspell - - nakedret - - prealloc - - revive - - staticcheck - - unconvert - - unparam - - unused - - logcheck - settings: - custom: - logcheck: - type: "module" - description: Checks Go logging calls for Kubernetes logging conventions. - depguard: - rules: - forbid-sort-pkg: - deny: - - pkg: sort - desc: Should be replaced with slices package - revive: - rules: - - name: comment-spacings - - name: import-shadowing - modernize: - disable: - - omitzero - - newexpr - exclusions: - generated: lax - rules: - - linters: - - lll - path: api/* - - linters: - - dupl - - lll - path: internal/* - paths: - - third_party$ - - builtin$ - - examples$ -formatters: - enable: - - gofmt - - goimports - exclusions: - generated: lax - paths: - - third_party$ - - builtin$ - - examples$ diff --git a/Makefile b/Makefile index 15b2f2a..5c25b31 100644 --- a/Makefile +++ b/Makefile @@ -94,18 +94,6 @@ test-e2e: setup-test-e2e manifests generate fmt vet ## Run the e2e tests. Expect cleanup-test-e2e: ## Tear down the Kind cluster used for e2e tests @$(KIND) delete cluster --name $(KIND_CLUSTER) -.PHONY: lint -lint: golangci-lint ## Run golangci-lint linter - "$(GOLANGCI_LINT)" run - -.PHONY: lint-fix -lint-fix: golangci-lint ## Run golangci-lint linter and perform fixes - "$(GOLANGCI_LINT)" run --fix - -.PHONY: lint-config -lint-config: golangci-lint ## Verify golangci-lint linter configuration - "$(GOLANGCI_LINT)" config verify - ##@ Build .PHONY: build @@ -188,7 +176,6 @@ KIND ?= kind KUSTOMIZE ?= $(LOCALBIN)/kustomize CONTROLLER_GEN ?= $(LOCALBIN)/controller-gen ENVTEST ?= $(LOCALBIN)/setup-envtest -GOLANGCI_LINT = $(LOCALBIN)/golangci-lint ## Tool Versions KUSTOMIZE_VERSION ?= v5.8.1 @@ -204,7 +191,6 @@ ENVTEST_K8S_VERSION ?= $(shell v='$(call gomodver,k8s.io/api)'; \ [ -n "$$v" ] || { echo "Set ENVTEST_K8S_VERSION manually (k8s.io/api replace has no tag)" >&2; exit 1; }; \ printf '%s\n' "$$v" | sed -E 's/^v?[0-9]+\.([0-9]+).*/1.\1/') -GOLANGCI_LINT_VERSION ?= v2.12.2 .PHONY: kustomize kustomize: $(KUSTOMIZE) ## Download kustomize locally if necessary. $(KUSTOMIZE): $(LOCALBIN) @@ -228,16 +214,6 @@ envtest: $(ENVTEST) ## Download setup-envtest locally if necessary. $(ENVTEST): $(LOCALBIN) $(call go-install-tool,$(ENVTEST),sigs.k8s.io/controller-runtime/tools/setup-envtest,$(ENVTEST_VERSION)) -.PHONY: golangci-lint -golangci-lint: $(GOLANGCI_LINT) ## Download golangci-lint locally if necessary. -$(GOLANGCI_LINT): $(LOCALBIN) - $(call go-install-tool,$(GOLANGCI_LINT),github.com/golangci/golangci-lint/v2/cmd/golangci-lint,$(GOLANGCI_LINT_VERSION)) - @test -f .custom-gcl.yml && { \ - echo "Building custom golangci-lint with plugins..." && \ - $(GOLANGCI_LINT) custom --destination $(LOCALBIN) --name golangci-lint-custom && \ - mv -f $(LOCALBIN)/golangci-lint-custom $(GOLANGCI_LINT); \ - } || true - # go-install-tool will 'go install' any package with custom target and name of binary, if it doesn't exist # $1 - target path with name of binary # $2 - package url which can be installed diff --git a/nix/default.nix b/nix/default.nix index 99ca6d5..9638c44 100644 --- a/nix/default.nix +++ b/nix/default.nix @@ -51,4 +51,30 @@ let --file $out ''; -in { inherit manager image image-arm64 sbom; } + # CI check: go fmt + go vet + unit tests (native only — arm64 cross tests cannot + # run on an amd64 host, so doCheck is NOT set in buildManager itself). + tests = (buildManager pkgs).overrideAttrs (_: { + pname = "cluster-api-janitor-openstack-tests"; + subPackages = []; # build all packages, not just cmd/ + doCheck = true; + checkPhase = '' + runHook preCheck + + bad=$(gofmt -l $(find . -name '*.go' \ + -not -path './vendor/*' -not -path './.git/*')) + if [ -n "$bad" ]; then + echo "Files not formatted with go fmt:" + printf '%s\n' $bad + exit 1 + fi + + go vet ./... + + go test -v $(go list ./... | grep -v '/test/e2e') + + runHook postCheck + ''; + installPhase = "touch $out"; + }); + +in { inherit manager image image-arm64 sbom tests; } From a681ba786ebd45cfe1ba4cb2380174794544128d Mon Sep 17 00:00:00 2001 From: Victor Hang Date: Fri, 10 Jul 2026 12:23:59 +0200 Subject: [PATCH 20/27] feat: allow userpass cred login, wider prefix selector --- cmd/main.go | 2 +- config/rbac/role.yaml | 45 +++++- .../controller/openstackcluster_controller.go | 2 +- .../openstackcluster_controller_test.go | 2 +- internal/openstack/cloud.go | 132 ++++++++++++++++-- internal/openstack/cloud_test.go | 93 +++++++++++- internal/openstack/resources.go | 7 +- internal/openstack/resources_test.go | 37 ++++- 8 files changed, 295 insertions(+), 25 deletions(-) diff --git a/cmd/main.go b/cmd/main.go index e2bc30f..33d347a 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -28,13 +28,13 @@ import ( "k8s.io/apimachinery/pkg/runtime" utilruntime "k8s.io/apimachinery/pkg/util/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" + infrav1 "sigs.k8s.io/cluster-api-provider-openstack/api/v1beta1" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/log/zap" "sigs.k8s.io/controller-runtime/pkg/metrics/filters" metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" "sigs.k8s.io/controller-runtime/pkg/webhook" - infrav1 "sigs.k8s.io/cluster-api-provider-openstack/api/v1beta1" "github.com/azimuth-cloud/cluster-api-janitor-openstack/internal/controller" // +kubebuilder:scaffold:imports diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 6cab0a0..b286585 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -1,11 +1,44 @@ +--- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: - labels: - app.kubernetes.io/name: cluster-api-janitor-openstack - app.kubernetes.io/managed-by: kustomize name: manager-role rules: -- apiGroups: [""] - resources: ["pods"] - verbs: ["get", "list", "watch"] +- apiGroups: + - "" + resources: + - events + verbs: + - create +- apiGroups: + - "" + resources: + - namespaces + verbs: + - list + - watch +- apiGroups: + - "" + resources: + - secrets + verbs: + - delete + - get +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - get + - list + - watch +- apiGroups: + - infrastructure.cluster.x-k8s.io + resources: + - openstackclusters + verbs: + - get + - list + - patch + - update + - watch diff --git a/internal/controller/openstackcluster_controller.go b/internal/controller/openstackcluster_controller.go index d190295..1a6045c 100644 --- a/internal/controller/openstackcluster_controller.go +++ b/internal/controller/openstackcluster_controller.go @@ -30,12 +30,12 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/tools/record" + infrav1 "sigs.k8s.io/cluster-api-provider-openstack/api/v1beta1" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/log" ctrlmetrics "sigs.k8s.io/controller-runtime/pkg/metrics" - infrav1 "sigs.k8s.io/cluster-api-provider-openstack/api/v1beta1" "github.com/azimuth-cloud/cluster-api-janitor-openstack/internal/openstack" ) diff --git a/internal/controller/openstackcluster_controller_test.go b/internal/controller/openstackcluster_controller_test.go index 0eeeaeb..13a3de6 100644 --- a/internal/controller/openstackcluster_controller_test.go +++ b/internal/controller/openstackcluster_controller_test.go @@ -13,11 +13,11 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" clientgoscheme "k8s.io/client-go/kubernetes/scheme" + infrav1 "sigs.k8s.io/cluster-api-provider-openstack/api/v1beta1" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" - infrav1 "sigs.k8s.io/cluster-api-provider-openstack/api/v1beta1" "github.com/azimuth-cloud/cluster-api-janitor-openstack/internal/controller" "github.com/azimuth-cloud/cluster-api-janitor-openstack/internal/openstack" diff --git a/internal/openstack/cloud.go b/internal/openstack/cloud.go index 639b4e5..eb3ca4f 100644 --- a/internal/openstack/cloud.go +++ b/internal/openstack/cloud.go @@ -11,8 +11,8 @@ import ( "io" "net/http" "regexp" - "time" "strings" + "time" "sigs.k8s.io/yaml" ) @@ -20,6 +20,11 @@ import ( const ( // KeepProperty is the OpenStack volume metadata key that marks a volume as user-kept. KeepProperty = "janitor.capi.azimuth-cloud.com/keep" + + // authTypeAppCred is the clouds.yaml auth_type for application credentials. + authTypeAppCred = "v3applicationcredential" + // authTypePassword is the clouds.yaml auth_type for username/password auth. + authTypePassword = "v3password" ) // AuthenticationError is returned when OpenStack authentication fails. @@ -65,6 +70,19 @@ type authBlock struct { AuthURL string `yaml:"auth_url" json:"auth_url"` ApplicationCredentialID string `yaml:"application_credential_id" json:"application_credential_id"` ApplicationCredentialSecret string `yaml:"application_credential_secret" json:"application_credential_secret"` + + // Username/password (v3password) fields. + Username string `yaml:"username" json:"username"` + UserID string `yaml:"user_id" json:"user_id"` + Password string `yaml:"password" json:"password"` + ProjectID string `yaml:"project_id" json:"project_id"` + ProjectName string `yaml:"project_name" json:"project_name"` + UserDomainName string `yaml:"user_domain_name" json:"user_domain_name"` + UserDomainID string `yaml:"user_domain_id" json:"user_domain_id"` + ProjectDomainName string `yaml:"project_domain_name" json:"project_domain_name"` + ProjectDomainID string `yaml:"project_domain_id" json:"project_domain_id"` + DomainName string `yaml:"domain_name" json:"domain_name"` + DomainID string `yaml:"domain_id" json:"domain_id"` } // parseCloudsYAML parses a clouds.yaml string into a cloudsFile. @@ -115,8 +133,9 @@ func Authenticate(ctx context.Context, cloudsYAML, cloudName, cacert string) (*S if !ok { return nil, fmt.Errorf("cloud %q not found in clouds.yaml", cloudName) } - if entry.AuthType != "v3applicationcredential" { - return nil, &UnsupportedAuthTypeError{AuthType: entry.AuthType} + authType, err := resolveAuthType(entry) + if err != nil { + return nil, err } tlsCfg := &tls.Config{} //nolint:gosec @@ -137,7 +156,7 @@ func Authenticate(ctx context.Context, cloudsYAML, cloudName, cacert string) (*S baseURL := authURLBase(entry.Auth.AuthURL) s := &Session{httpClient: hc} - if err := s.getToken(ctx, baseURL, entry.Auth); err != nil { + if err := s.getToken(ctx, baseURL, authType, entry.Auth); err != nil { if isHTTP(err, http.StatusNotFound) { return s, nil // deleted appcred case: unauthenticated, no fatal error } @@ -149,8 +168,95 @@ func Authenticate(ctx context.Context, cloudsYAML, cloudName, cacert string) (*S return s, nil } -func (s *Session) getToken(ctx context.Context, baseURL string, auth authBlock) error { - body, _ := json.Marshal(map[string]any{ +// resolveAuthType determines the effective auth type for a cloud entry. +// It honours an explicit auth_type, and otherwise infers it from the auth +// block (application credential fields imply v3applicationcredential, a +// username/user_id implies v3password) to match common clouds.yaml usage +// where auth_type is omitted. +func resolveAuthType(entry cloudEntry) (string, error) { + switch entry.AuthType { + case authTypeAppCred, authTypePassword: + return entry.AuthType, nil + case "", "password": + // Infer from the auth block for empty or ambiguous "password" values. + if entry.Auth.ApplicationCredentialID != "" || entry.Auth.ApplicationCredentialSecret != "" { + return authTypeAppCred, nil + } + if entry.Auth.Username != "" || entry.Auth.UserID != "" { + return authTypePassword, nil + } + return "", &UnsupportedAuthTypeError{AuthType: entry.AuthType} + default: + return "", &UnsupportedAuthTypeError{AuthType: entry.AuthType} + } +} + +// passwordScope builds the Keystone auth scope from the auth block, preferring +// project scoping and falling back to domain scoping when set. +func passwordScope(auth authBlock) map[string]any { + if auth.ProjectID != "" || auth.ProjectName != "" { + project := map[string]any{} + if auth.ProjectID != "" { + project["id"] = auth.ProjectID + } else { + project["name"] = auth.ProjectName + domain := map[string]any{} + switch { + case auth.ProjectDomainID != "": + domain["id"] = auth.ProjectDomainID + case auth.ProjectDomainName != "": + domain["name"] = auth.ProjectDomainName + case auth.UserDomainID != "": + domain["id"] = auth.UserDomainID + case auth.UserDomainName != "": + domain["name"] = auth.UserDomainName + } + if len(domain) > 0 { + project["domain"] = domain + } + } + return map[string]any{"project": project} + } + if auth.DomainID != "" { + return map[string]any{"domain": map[string]any{"id": auth.DomainID}} + } + if auth.DomainName != "" { + return map[string]any{"domain": map[string]any{"name": auth.DomainName}} + } + return nil +} + +// passwordTokenBody builds the Keystone token request body for v3password auth. +func passwordTokenBody(auth authBlock) map[string]any { + user := map[string]any{"password": auth.Password} + if auth.UserID != "" { + user["id"] = auth.UserID + } else { + user["name"] = auth.Username + domain := map[string]any{} + if auth.UserDomainID != "" { + domain["id"] = auth.UserDomainID + } else if auth.UserDomainName != "" { + domain["name"] = auth.UserDomainName + } + if len(domain) > 0 { + user["domain"] = domain + } + } + identity := map[string]any{ + "methods": []string{"password"}, + "password": map[string]any{"user": user}, + } + authReq := map[string]any{"identity": identity} + if scope := passwordScope(auth); scope != nil { + authReq["scope"] = scope + } + return map[string]any{"auth": authReq} +} + +// appCredTokenBody builds the Keystone token request body for application credential auth. +func appCredTokenBody(auth authBlock) map[string]any { + return map[string]any{ "auth": map[string]any{ "identity": map[string]any{ "methods": []string{"application_credential"}, @@ -160,7 +266,17 @@ func (s *Session) getToken(ctx context.Context, baseURL string, auth authBlock) }, }, }, - }) + } +} + +func (s *Session) getToken(ctx context.Context, baseURL, authType string, auth authBlock) error { + var payload map[string]any + if authType == authTypePassword { + payload = passwordTokenBody(auth) + } else { + payload = appCredTokenBody(auth) + } + body, _ := json.Marshal(payload) req, err := http.NewRequestWithContext(ctx, http.MethodPost, baseURL+"/v3/auth/tokens", strings.NewReader(string(body))) if err != nil { @@ -290,7 +406,7 @@ func (s *Session) doDelete(ctx context.Context, url string) error { type httpStatusError struct{ code int } -func (e *httpStatusError) Error() string { return fmt.Sprintf("HTTP %d", e.code) } +func (e *httpStatusError) Error() string { return fmt.Sprintf("HTTP %d", e.code) } func (e *httpStatusError) StatusCode() int { return e.code } func isHTTP(err error, code int) bool { diff --git a/internal/openstack/cloud_test.go b/internal/openstack/cloud_test.go index 91e9bdb..9992b41 100644 --- a/internal/openstack/cloud_test.go +++ b/internal/openstack/cloud_test.go @@ -21,6 +21,7 @@ type keystoneServer struct { tokenUserID string catalogStatus int catalog map[string]any + lastTokenBody map[string]any } func newKeystoneServer(t *testing.T) *keystoneServer { @@ -37,6 +38,10 @@ func newKeystoneServer(t *testing.T) *keystoneServer { w.WriteHeader(http.StatusMethodNotAllowed) return } + var reqBody map[string]any + if err := json.NewDecoder(r.Body).Decode(&reqBody); err == nil { + ks.lastTokenBody = reqBody + } if ks.tokenStatus >= 400 { w.WriteHeader(ks.tokenStatus) return @@ -198,12 +203,12 @@ func TestAuthenticate_SuccessfulAuthentication(t *testing.T) { } // Scenario: Authentication with unsupported type -// Given a clouds.yaml with auth_type "password" +// Given a clouds.yaml with an unsupported auth_type "token" // When the operator attempts to create a Cloud client // Then an UnsupportedAuthTypeError is raised func TestAuthenticate_UnsupportedAuthType(t *testing.T) { ks := newKeystoneServer(t) - clouds := buildCloudsYAML(ks.URL, "password") + clouds := buildCloudsYAML(ks.URL, "token") _, err := openstack.Authenticate(context.Background(), clouds, "openstack", "") @@ -214,8 +219,88 @@ func TestAuthenticate_UnsupportedAuthType(t *testing.T) { if !errorAs(err, &target) { t.Fatalf("expected *UnsupportedAuthTypeError, got %T: %v", err, err) } - if target.AuthType != "password" { - t.Errorf("expected AuthType %q, got %q", "password", target.AuthType) + if target.AuthType != "token" { + t.Errorf("expected AuthType %q, got %q", "token", target.AuthType) + } +} + +// ── Password (v3password) authentication ────────────────────────────────── + +func buildPasswordCloudsYAML(authURL, authType string) string { + return fmt.Sprintf(` +clouds: + openstack: + auth_type: %s + auth: + auth_url: %s + username: alice + password: s3cret + project_id: proj-123 + user_domain_name: Default + region_name: RegionOne + interface: public +`, authType, authURL) +} + +// Scenario: Successful authentication via username/password +func TestAuthenticate_Password_Successful(t *testing.T) { + ks := newKeystoneServer(t) + ks.catalog = catalogWith( + serviceEntry("compute", + endpoint("public", "RegionOne", "http://compute.example.com"), + ), + ) + + clouds := buildPasswordCloudsYAML(ks.URL, "v3password") + session, err := openstack.Authenticate(context.Background(), clouds, "openstack", "") + if err != nil { + t.Fatalf("expected no error, got: %v", err) + } + if !session.IsAuthenticated() { + t.Error("expected session to be authenticated") + } + + // Verify the request used the password method with a project scope. + auth, _ := ks.lastTokenBody["auth"].(map[string]any) + identity, _ := auth["identity"].(map[string]any) + methods, _ := identity["methods"].([]any) + if len(methods) != 1 || methods[0] != "password" { + t.Errorf("expected password method, got %v", methods) + } + pw, _ := identity["password"].(map[string]any) + user, _ := pw["user"].(map[string]any) + if user["name"] != "alice" || user["password"] != "s3cret" { + t.Errorf("unexpected user block: %v", user) + } + scope, _ := auth["scope"].(map[string]any) + project, _ := scope["project"].(map[string]any) + if project["id"] != "proj-123" { + t.Errorf("expected project id proj-123, got %v", project["id"]) + } +} + +// Scenario: auth_type omitted but username present is inferred as v3password +func TestAuthenticate_Password_InferredFromUsername(t *testing.T) { + ks := newKeystoneServer(t) + ks.catalog = catalogWith( + serviceEntry("compute", + endpoint("public", "RegionOne", "http://compute.example.com"), + ), + ) + + clouds := buildPasswordCloudsYAML(ks.URL, "") + session, err := openstack.Authenticate(context.Background(), clouds, "openstack", "") + if err != nil { + t.Fatalf("expected no error, got: %v", err) + } + if !session.IsAuthenticated() { + t.Error("expected session to be authenticated") + } + auth, _ := ks.lastTokenBody["auth"].(map[string]any) + identity, _ := auth["identity"].(map[string]any) + methods, _ := identity["methods"].([]any) + if len(methods) != 1 || methods[0] != "password" { + t.Errorf("expected inferred password method, got %v", methods) } } diff --git a/internal/openstack/resources.go b/internal/openstack/resources.go index 268b1ca..d6499c8 100644 --- a/internal/openstack/resources.go +++ b/internal/openstack/resources.go @@ -18,7 +18,7 @@ func (s *Session) DeleteFloatingIPs(ctx context.Context, logger logr.Logger, clu if err != nil { return err } - prefix := "Floating IP for Kubernetes external service" + prefix := "Floating IP for Kubernetes" suffix := "from cluster " + cluster type fip struct { @@ -256,6 +256,11 @@ func (s *Session) DeleteAppCredential(ctx context.Context, logger logr.Logger, c if err != nil { return err } + if appcredID == "" { + // Password (v3password) auth has no application credential to delete. + logger.Info("no application credential in use, skipping deletion") + return nil + } target := strings.TrimRight(identityURL, "/") + "/v3/users/" + s.userID + "/application_credentials/" + appcredID req, err := newDeleteRequest(ctx, target, s.token) if err != nil { diff --git a/internal/openstack/resources_test.go b/internal/openstack/resources_test.go index 6386b29..8d702c5 100644 --- a/internal/openstack/resources_test.go +++ b/internal/openstack/resources_test.go @@ -351,10 +351,10 @@ func TestDeleteFloatingIPs_Filtering(t *testing.T) { func TestDeleteFloatingIPs_MultipleIPsPartialMatch(t *testing.T) { srv := newNetworkTestServer(t) fips := []fipRecord{ - {ID: "fip-001", Description: fipDesc("mycluster")}, // match - {ID: "fip-002", Description: fipDesc("othercluster")}, // no match + {ID: "fip-001", Description: fipDesc("mycluster")}, // match + {ID: "fip-002", Description: fipDesc("othercluster")}, // no match {ID: "fip-003", Description: "Some other description"}, // no match - {ID: "fip-004", Description: fipDesc("mycluster")}, // match + {ID: "fip-004", Description: fipDesc("mycluster")}, // match } srv.fipLists = [][]fipRecord{fips, {}} @@ -1856,6 +1856,37 @@ func TestDeleteAppCredential_HTTP500_PropagatesError(t *testing.T) { } } +// Scenario: Password auth has no application credential → deletion is a no-op +func TestDeleteAppCredential_PasswordAuth_NoOp(t *testing.T) { + srv := newIdentityTestServer(t) + session, _ := srv.authenticate(t) + + passwordYAML := fmt.Sprintf(` +clouds: + openstack: + auth_type: v3password + auth: + auth_url: %s + username: alice + password: s3cret + project_id: proj-123 + user_domain_name: Default + interface: public + region_name: RegionOne +`, srv.URL) + + if err := session.DeleteAppCredential(context.Background(), logr.Discard(), passwordYAML, "openstack"); err != nil { + t.Fatalf("expected nil for password auth (no appcred), got: %v", err) + } + + srv.mu.Lock() + deleted := srv.deletedAppcredID + srv.mu.Unlock() + if deleted != "" { + t.Errorf("expected no DELETE call, but appcred %q was deleted", deleted) + } +} + // Scenario: No "identity" endpoint in catalog → CatalogError func TestDeleteAppCredential_NoIdentityEndpoint_ReturnsCatalogError(t *testing.T) { ks := newKeystoneServer(t) From 10e0da12f8bc35704f4a9d9bbb39a2cee8936252 Mon Sep 17 00:00:00 2001 From: Victor Hang Date: Fri, 10 Jul 2026 13:24:56 +0200 Subject: [PATCH 21/27] fix: wait for deletion (retry) --- internal/openstack/cloud.go | 12 +++ internal/openstack/resources.go | 104 +++++++++++++++++--------- internal/openstack/resources_test.go | 40 +++++++++- internal/openstack/robustness_test.go | 2 + 4 files changed, 118 insertions(+), 40 deletions(-) diff --git a/internal/openstack/cloud.go b/internal/openstack/cloud.go index eb3ca4f..76ee009 100644 --- a/internal/openstack/cloud.go +++ b/internal/openstack/cloud.go @@ -108,6 +108,18 @@ type Session struct { endpoints map[string]string httpClient *http.Client authenticated bool + // SleepFunc is called instead of time.Sleep for polling waits. + // A nil value defaults to time.Sleep. + SleepFunc func(time.Duration) +} + +// sleep calls SleepFunc if set, otherwise time.Sleep. +func (s *Session) sleep(d time.Duration) { + if s.SleepFunc != nil { + s.SleepFunc(d) + } else { + time.Sleep(d) + } } // IsAuthenticated reports whether the session has a valid token and catalog. diff --git a/internal/openstack/resources.go b/internal/openstack/resources.go index d6499c8..e16370b 100644 --- a/internal/openstack/resources.go +++ b/internal/openstack/resources.go @@ -7,6 +7,7 @@ import ( "net/http" "net/url" "strings" + "time" "github.com/go-logr/logr" ) @@ -46,14 +47,14 @@ func (s *Session) DeleteFloatingIPs(ctx context.Context, logger logr.Logger, clu } } if deleted { - remaining, err := listPages[fip](ctx, s, networkURL+"/v2.0/floatingips", "floatingips") - if err != nil { - return err + listFIPs := func() ([]fip, error) { + return listPages[fip](ctx, s, networkURL+"/v2.0/floatingips", "floatingips") } - for _, f := range remaining { - if strings.HasPrefix(f.Description, prefix) && strings.HasSuffix(f.Description, suffix) { - return fmt.Errorf("floating IPs still present for cluster %s", cluster) - } + matchFIP := func(f fip) bool { + return strings.HasPrefix(f.Description, prefix) && strings.HasSuffix(f.Description, suffix) + } + if err := waitForDeletion(ctx, s, logger, listFIPs, matchFIP, "floating IPs for cluster "+cluster); err != nil { + return err } } logger.Info("deleted floating IPs for LoadBalancer services") @@ -95,15 +96,15 @@ func (s *Session) DeleteLoadBalancers(ctx context.Context, logger logr.Logger, c } } if deleted { - remaining, err := listPages[lb](ctx, s, lbURL+"/v2/lbaas/loadbalancers", "loadbalancers") - if err != nil { - logger.Error(err, "failed to verify LB deletion") - return nil + listLBs := func() ([]lb, error) { + return listPages[lb](ctx, s, lbURL+"/v2/lbaas/loadbalancers", "loadbalancers") } - for _, l := range remaining { - if strings.HasPrefix(l.Name, kubePrefix) { - return fmt.Errorf("load balancers still present for cluster %s", cluster) - } + matchLB := func(l lb) bool { + return strings.HasPrefix(l.Name, kubePrefix) + } + if err := waitForDeletion(ctx, s, logger, listLBs, matchLB, "load balancers for cluster "+cluster); err != nil { + logger.Error(err, "failed to verify LB deletion after polling") + return err } } logger.Info("deleted load balancers for LoadBalancer services") @@ -142,14 +143,14 @@ func (s *Session) DeleteSecurityGroups(ctx context.Context, logger logr.Logger, } } if deleted { - remaining, err := listPages[sg](ctx, s, networkURL+"/v2.0/security-groups", "security_groups") - if err != nil { - return err + listSGs := func() ([]sg, error) { + return listPages[sg](ctx, s, networkURL+"/v2.0/security-groups", "security_groups") } - for _, g := range remaining { - if strings.HasPrefix(g.Description, "Security Group for") && strings.HasSuffix(g.Description, sgSuffix) { - return fmt.Errorf("security groups still present for cluster %s", cluster) - } + matchSG := func(g sg) bool { + return strings.HasPrefix(g.Description, "Security Group for") && strings.HasSuffix(g.Description, sgSuffix) + } + if err := waitForDeletion(ctx, s, logger, listSGs, matchSG, "security groups for cluster "+cluster); err != nil { + return err } } logger.Info("deleted security groups for LoadBalancer services") @@ -187,14 +188,14 @@ func (s *Session) DeleteSnapshots(ctx context.Context, logger logr.Logger, clust } } if deleted { - remaining, err := s.listVolumeItems(ctx, cinderURL+"/snapshots/detail", "snapshots") - if err != nil { - return err + listSnaps := func() ([]volumeItem, error) { + return s.listVolumeItems(ctx, cinderURL+"/snapshots/detail", "snapshots") } - for _, snap := range remaining { - if snap.Metadata["cinder.csi.openstack.org/cluster"] == cluster { - return fmt.Errorf("snapshots still present for cluster %s", cluster) - } + matchSnap := func(snap volumeItem) bool { + return snap.Metadata["cinder.csi.openstack.org/cluster"] == cluster + } + if err := waitForDeletion(ctx, s, logger, listSnaps, matchSnap, "snapshots for cluster "+cluster); err != nil { + return err } } logger.Info("deleted snapshots for persistent volume claims") @@ -231,15 +232,15 @@ func (s *Session) DeleteVolumes(ctx context.Context, logger logr.Logger, cluster } } if deleted { - remaining, err := s.listVolumeItems(ctx, cinderURL+"/volumes/detail", "volumes") - if err != nil { - return err + listVols := func() ([]volumeItem, error) { + return s.listVolumeItems(ctx, cinderURL+"/volumes/detail", "volumes") } - for _, vol := range remaining { - if vol.Metadata["cinder.csi.openstack.org/cluster"] == cluster && - vol.Metadata[KeepProperty] != "true" { - return fmt.Errorf("volumes still present for cluster %s", cluster) - } + matchVol := func(vol volumeItem) bool { + return vol.Metadata["cinder.csi.openstack.org/cluster"] == cluster && + vol.Metadata[KeepProperty] != "true" + } + if err := waitForDeletion(ctx, s, logger, listVols, matchVol, "volumes for cluster "+cluster); err != nil { + return err } } logger.Info("deleted volumes for persistent volume claims") @@ -312,6 +313,35 @@ func newDeleteRequest(ctx context.Context, target, token string) (*http.Request, return req, nil } +// waitForDeletion polls a list function until no items match the predicate, +// or the maximum number of attempts is exceeded. This avoids failing +// immediately when OpenStack has accepted a DELETE but the resource has +// not yet been removed from list results. +func waitForDeletion[T any](ctx context.Context, s *Session, logger logr.Logger, listFunc func() ([]T, error), matchFunc func(T) bool, desc string) error { + const ( + maxPollAttempts = 6 + pollInterval = 5 * time.Second + ) + for attempt := 0; attempt < maxPollAttempts; attempt++ { + s.sleep(pollInterval) + items, err := listFunc() + if err != nil { + return err + } + remaining := 0 + for _, item := range items { + if matchFunc(item) { + remaining++ + } + } + if remaining == 0 { + return nil + } + logger.Info(desc+" still present, retrying", "remaining", remaining, "attempt", attempt+1, "maxAttempts", maxPollAttempts) + } + return fmt.Errorf("%s still present", desc) +} + // listPages fetches all pages of a paginated OpenStack list endpoint and // decodes items into T using the given JSON key. func listPages[T any](ctx context.Context, s *Session, endpoint, key string) ([]T, error) { diff --git a/internal/openstack/resources_test.go b/internal/openstack/resources_test.go index 8d702c5..2878cfa 100644 --- a/internal/openstack/resources_test.go +++ b/internal/openstack/resources_test.go @@ -9,6 +9,7 @@ import ( "strings" "sync" "testing" + "time" "github.com/go-logr/logr" @@ -141,9 +142,7 @@ clouds: if err != nil { t.Fatalf("authenticate: %v", err) } - if !session.IsAuthenticated() { - t.Fatal("expected authenticated session with network endpoint") - } + session.SleepFunc = func(d time.Duration) {} return session } @@ -279,6 +278,7 @@ clouds: if err != nil { t.Fatalf("authenticate: %v", err) } + session.SleepFunc = func(d time.Duration) {} return session } @@ -502,6 +502,36 @@ func TestDeleteFloatingIPs_StillPresentAfterDeletion_ReturnsError(t *testing.T) } } +// Scenario: FIP disappears during polling → polling succeeds without error +func TestDeleteFloatingIPs_SlowDeletion_PollingSucceeds(t *testing.T) { + srv := newNetworkTestServer(t) + fip := fipRecord{ID: "fip-slow", Description: fipDesc("mycluster")} + // idx 0: pre-delete list (FIP present) + // idx 1: first poll (FIP still present — OpenStack PENDING_DELETE) + // idx 2: second poll (FIP gone — deletion complete) + srv.fipLists = [][]fipRecord{{fip}, {fip}, {}} + + session := srv.authenticate(t) + err := session.DeleteFloatingIPs(context.Background(), logr.Discard(), "mycluster") + + if err != nil { + t.Fatalf("expected no error when FIP disappears during polling, got: %v", err) + } + + srv.mu.Lock() + deletedIDs := srv.deletedFIPs + getCount := srv.fipGetCount + srv.mu.Unlock() + + if len(deletedIDs) != 1 { + t.Errorf("expected 1 FIP deleted, got %d: %v", len(deletedIDs), deletedIDs) + } + // 1 pre-delete list + 2 polls = 3 GET calls + if getCount != 3 { + t.Errorf("expected 3 GET calls (1 list + 2 polls), got %d", getCount) + } +} + // Scenario: No matching FIP → no deletion, no verification func TestDeleteFloatingIPs_NothingToDelete_NoVerification(t *testing.T) { srv := newNetworkTestServer(t) @@ -904,6 +934,7 @@ clouds: if !session.IsAuthenticated() { t.Fatal("expected authenticated session with network endpoint") } + session.SleepFunc = func(d time.Duration) {} return session } @@ -1238,6 +1269,7 @@ clouds: if !session.IsAuthenticated() { t.Fatal("expected authenticated session with volumev3 endpoint") } + session.SleepFunc = func(d time.Duration) {} return session } @@ -1545,6 +1577,7 @@ clouds: if !session.IsAuthenticated() { t.Fatal("expected authenticated session with volumev3 endpoint") } + session.SleepFunc = func(d time.Duration) {} return session } @@ -1797,6 +1830,7 @@ clouds: if !session.IsAuthenticated() { t.Fatal("expected authenticated session with identity endpoint") } + session.SleepFunc = func(d time.Duration) {} return session, cloudsYAML } diff --git a/internal/openstack/robustness_test.go b/internal/openstack/robustness_test.go index d1c1709..1f227c6 100644 --- a/internal/openstack/robustness_test.go +++ b/internal/openstack/robustness_test.go @@ -8,6 +8,7 @@ import ( "net/http/httptest" "sync" "testing" + "time" "github.com/go-logr/logr" @@ -122,6 +123,7 @@ clouds: if !session.IsAuthenticated() { t.Fatal("expected authenticated session") } + session.SleepFunc = func(d time.Duration) {} return session } From c3d2f4bde34402feb33d1aa112139055e34c49f2 Mon Sep 17 00:00:00 2001 From: Mathieu Grzybek Date: Fri, 10 Jul 2026 13:38:58 +0200 Subject: [PATCH 22/27] chore(build): gofmt imports, pin Nix vendorHash, document CI test target Reorder infrav1 imports per gofmt/goimports convention, set the real vendorHash in nix/default.nix (was fakeHash), and update the README to document `nix-build nix -A tests` and the macOS/Linux binary linkage differences. Signed-off-by: Mathieu Grzybek --- README.md | 31 +++++++++++++++++++++++-------- nix/default.nix | 4 ++-- 2 files changed, 25 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 331213c..5de8141 100644 --- a/README.md +++ b/README.md @@ -140,7 +140,6 @@ required. ```sh go fmt ./... go vet ./... -golangci-lint run ./... # config in .golangci.yml ``` ### Makefile targets @@ -151,7 +150,7 @@ make generate # regenerate DeepCopy methods make manifests # regenerate CRD/RBAC YAML make fmt # go fmt make vet # go vet -make test # go test with race detector +make test # go test (excludes e2e) make build # go build ./cmd/main.go ``` @@ -168,22 +167,38 @@ The Dockerfile uses a multi-stage build: `golang:1.26` builder → ### Nix (reproducible, multi-arch + SBOM) -CI uses `nix-build` for reproducible images and SBOM generation: +CI uses `nix-build` for reproducible builds. The `tests` derivation runs +`go fmt`, `go vet`, and the full unit-test suite inside the Nix sandbox — no +external toolchain needed: ```sh +# CI check: go fmt + go vet + 108 unit tests +nix-build nix -A tests + +# Build the manager binary only +nix-build nix -A manager + # Build the amd64 OCI image nix-build nix -A image -# Build the arm64 OCI image (cross-compiled) +# Build the arm64 OCI image (cross-compiled from amd64) nix-build nix -A image-arm64 # Generate the CycloneDX SBOM -nix-build nix -A sbom && cat result | python3 -m json.tool | grep bomFormat +nix-build nix -A sbom ``` -> `nix/nixpkgs.nix` pins `nixos-26.05` (required for Go 1.26). -> On the first run, `nix-build nix -A manager` will fail with the real -> `vendorHash` to substitute in `nix/default.nix`. +> **`nix/nixpkgs.nix`** pins `nixos-26.05` (Go 1.26+). +> **`vendorHash`** in `nix/default.nix` is set to `sha256-5p5z+fzRkBk6rIb3DWwA3jsF4MdMVAwKHz7xza09fCc=` +> (run `nix-build nix -A manager` after any `go.mod` change — the build will +> fail and print the new hash to substitute). + +**Binary linkage note**: on macOS the local build produces a darwin/arm64 Mach-O +(dynamically linked against system libraries — normal for Go on Darwin). The +arm64 image produced via `pkgsCross.aarch64-multiplatform` contains a Linux ELF +dynamically linked against glibc; `buildLayeredImage` automatically includes the +full Nix closure (glibc and its dependencies) so the image is self-contained and +runs correctly in Kubernetes. ## Observability diff --git a/nix/default.nix b/nix/default.nix index 9638c44..77b9bd2 100644 --- a/nix/default.nix +++ b/nix/default.nix @@ -7,10 +7,10 @@ let version = "0.0.0-dev"; src = ../.; subPackages = [ "cmd" ]; - CGO_ENABLED = "0"; + env.CGO_ENABLED = "0"; ldflags = [ "-s" "-w" ]; # Run `nix-build nix -A manager` once; it will fail and print the real hash. - vendorHash = pkgs.lib.fakeHash; + vendorHash = "sha256-5p5z+fzRkBk6rIb3DWwA3jsF4MdMVAwKHz7xza09fCc="; postInstall = '' mv $out/bin/cmd $out/bin/manager ''; From 925b7d4506937bff81cb39c854679877a0e8f0ed Mon Sep 17 00:00:00 2001 From: Mathieu Grzybek Date: Fri, 10 Jul 2026 15:21:11 +0200 Subject: [PATCH 23/27] fix: correct waitForDeletion regressions from the retry refactor Restore non-fatal handling of LB verification-list errors (Octavia is known to be slower/less reliable, per PR #261), drop the unconditional 5s delay before the first deletion check, and stop the polling loop promptly on context cancellation. Also fill in two test helpers that had lost their IsAuthenticated() assertion, and document the polling mechanism in ROADMAP.md. Signed-off-by: Mathieu Grzybek --- ROADMAP.md | 30 ++++++++++ internal/openstack/resources.go | 25 +++++++- internal/openstack/resources_test.go | 86 +++++++++++++++++++++++++++- 3 files changed, 135 insertions(+), 6 deletions(-) diff --git a/ROADMAP.md b/ROADMAP.md index 41bc943..6eaf9ac 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -210,8 +210,26 @@ Feature: Floating IP Deletion Given a FIP deletion returns HTTP 500 When the purge attempts to delete the FIP Then an exception is propagated + + Scenario: FIP deletion confirmed via polling + Given a FIP still appears in a first verification listing (OpenStack PENDING_DELETE) + When the purge polls again shortly after + And the FIP has disappeared + Then no error is raised + + Scenario: FIP still present after exhausting verification attempts + Given a FIP still appears in every verification listing + When the purge exhausts its polling attempts + Then an error is returned mentioning the cluster name ``` +> Deletion verification for FIPs, LBs, security groups, volumes and snapshots +> (Epics 2 to 6) shares a common polling mechanism: verify immediately after +> issuing the deletes, then retry a bounded number of times with a fixed +> delay between attempts if the resource is still listed. This absorbs +> OpenStack's eventual consistency (`PENDING_DELETE` states) without +> incurring a wait when nothing needs one. + --- ### Epic 3 — Octavia Load Balancer Cleanup @@ -251,8 +269,20 @@ Feature: Identifying Azimuth Load Balancers Then an ERROR log is emitted with the HTTP code And no exception is propagated And a warning indicates that LBs may remain + + Scenario: HTTP error while verifying LB deletion after polling + Given LBs were deleted for cluster "mycluster" + And the Octavia API returns an HTTP error while verifying their deletion + When the verification polling exhausts its attempts due to the error + Then an ERROR log is emitted + And no exception is propagated ``` +> Unlike the other resource types, LB verification failures stay non-fatal +> (logged only) in both cases above — before and after the deletes are +> issued — since Octavia listing is known to be slower/less reliable than +> Neutron or Cinder (see PR #261). + #### US3.3 — Delete Load Balancers with Cascade ```gherkin diff --git a/internal/openstack/resources.go b/internal/openstack/resources.go index e16370b..d549c93 100644 --- a/internal/openstack/resources.go +++ b/internal/openstack/resources.go @@ -3,6 +3,7 @@ package openstack import ( "context" "encoding/json" + "errors" "fmt" "net/http" "net/url" @@ -103,8 +104,11 @@ func (s *Session) DeleteLoadBalancers(ctx context.Context, logger logr.Logger, c return strings.HasPrefix(l.Name, kubePrefix) } if err := waitForDeletion(ctx, s, logger, listLBs, matchLB, "load balancers for cluster "+cluster); err != nil { + var stillPresent *stillPresentError + if errors.As(err, &stillPresent) { + return err + } logger.Error(err, "failed to verify LB deletion after polling") - return err } } logger.Info("deleted load balancers for LoadBalancer services") @@ -313,6 +317,14 @@ func newDeleteRequest(ctx context.Context, target, token string) (*http.Request, return req, nil } +// stillPresentError indicates that a resource was still present after +// waitForDeletion exhausted its polling attempts, as opposed to an error +// from the underlying listFunc itself. Callers use errors.As to tell the two +// apart, since some resource types treat listing failures as non-fatal. +type stillPresentError struct{ desc string } + +func (e *stillPresentError) Error() string { return e.desc + " still present" } + // waitForDeletion polls a list function until no items match the predicate, // or the maximum number of attempts is exceeded. This avoids failing // immediately when OpenStack has accepted a DELETE but the resource has @@ -323,7 +335,14 @@ func waitForDeletion[T any](ctx context.Context, s *Session, logger logr.Logger, pollInterval = 5 * time.Second ) for attempt := 0; attempt < maxPollAttempts; attempt++ { - s.sleep(pollInterval) + if attempt > 0 { + s.sleep(pollInterval) + } + select { + case <-ctx.Done(): + return ctx.Err() + default: + } items, err := listFunc() if err != nil { return err @@ -339,7 +358,7 @@ func waitForDeletion[T any](ctx context.Context, s *Session, logger logr.Logger, } logger.Info(desc+" still present, retrying", "remaining", remaining, "attempt", attempt+1, "maxAttempts", maxPollAttempts) } - return fmt.Errorf("%s still present", desc) + return &stillPresentError{desc: desc} } // listPages fetches all pages of a paginated OpenStack list endpoint and diff --git a/internal/openstack/resources_test.go b/internal/openstack/resources_test.go index 2878cfa..3d507aa 100644 --- a/internal/openstack/resources_test.go +++ b/internal/openstack/resources_test.go @@ -3,6 +3,7 @@ package openstack_test import ( "context" "encoding/json" + "errors" "fmt" "net/http" "net/http/httptest" @@ -142,6 +143,9 @@ clouds: if err != nil { t.Fatalf("authenticate: %v", err) } + if !session.IsAuthenticated() { + t.Fatal("expected authenticated session with network endpoint") + } session.SleepFunc = func(d time.Duration) {} return session } @@ -166,7 +170,8 @@ type lbTestServer struct { mu sync.Mutex lbLists [][]lbRecord lbGetCount int - listStatusOverride int // if non-zero, all GET /v2/lbaas/loadbalancers return this status + listStatusOverride int // if non-zero, GET /v2/lbaas/loadbalancers returns this status + listFailFrom int // 1-based GET call number from which listStatusOverride applies; 0 = from the first call deleteStatus map[string]int deletedLBs []string deleteCascade []string // cascade query param value per DELETE call @@ -219,8 +224,10 @@ func newLBTestServer(t *testing.T) *lbTestServer { } srv.mu.Lock() override := srv.listStatusOverride + failFrom := srv.listFailFrom srv.lbGetCount++ - idx := srv.lbGetCount - 1 + callNum := srv.lbGetCount + idx := callNum - 1 var list []lbRecord if len(srv.lbLists) > 0 { if idx < len(srv.lbLists) { @@ -230,7 +237,7 @@ func newLBTestServer(t *testing.T) *lbTestServer { } } srv.mu.Unlock() - if override != 0 { + if override != 0 && (failFrom == 0 || callNum >= failFrom) { w.WriteHeader(override) return } @@ -278,6 +285,9 @@ clouds: if err != nil { t.Fatalf("authenticate: %v", err) } + if !session.IsAuthenticated() { + t.Fatal("expected authenticated session with load-balancer endpoint") + } session.SleepFunc = func(d time.Duration) {} return session } @@ -502,6 +512,33 @@ func TestDeleteFloatingIPs_StillPresentAfterDeletion_ReturnsError(t *testing.T) } } +// Scenario: context is canceled while polling → polling stops immediately, +// without an extra wasted list round-trip +func TestDeleteFloatingIPs_ContextCanceledDuringPolling_StopsImmediately(t *testing.T) { + srv := newNetworkTestServer(t) + fip := fipRecord{ID: "fip-persistent", Description: fipDesc("mycluster")} + srv.fipLists = [][]fipRecord{{fip}, {fip}, {fip}, {fip}} // always present + + session := srv.authenticate(t) + ctx, cancel := context.WithCancel(context.Background()) + session.SleepFunc = func(d time.Duration) { cancel() } + + err := session.DeleteFloatingIPs(ctx, logr.Discard(), "mycluster") + + if !errors.Is(err, context.Canceled) { + t.Fatalf("expected context.Canceled, got: %v", err) + } + srv.mu.Lock() + getCount := srv.fipGetCount + srv.mu.Unlock() + // 1 pre-delete list + 1 poll (attempt 0, before any sleep) = 2 GET calls. + // The sleep before attempt 1 cancels the context, so attempt 1's list call + // must never happen. + if getCount != 2 { + t.Errorf("expected exactly 2 GET calls (no wasted round-trip after cancellation), got %d", getCount) + } +} + // Scenario: FIP disappears during polling → polling succeeds without error func TestDeleteFloatingIPs_SlowDeletion_PollingSucceeds(t *testing.T) { srv := newNetworkTestServer(t) @@ -532,6 +569,25 @@ func TestDeleteFloatingIPs_SlowDeletion_PollingSucceeds(t *testing.T) { } } +// Scenario: FIP already gone on the first verification check → no wait incurred +func TestDeleteFloatingIPs_ImmediateVerification_NoSleep(t *testing.T) { + srv := newNetworkTestServer(t) + fip := fipRecord{ID: "fip-fast", Description: fipDesc("mycluster")} + srv.fipLists = [][]fipRecord{{fip}, {}} // gone by the first verification check + + session := srv.authenticate(t) + var sleepCalls int + session.SleepFunc = func(d time.Duration) { sleepCalls++ } + err := session.DeleteFloatingIPs(context.Background(), logr.Discard(), "mycluster") + + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if sleepCalls != 0 { + t.Errorf("expected no sleep when the resource is already gone, got %d sleep calls", sleepCalls) + } +} + // Scenario: No matching FIP → no deletion, no verification func TestDeleteFloatingIPs_NothingToDelete_NoVerification(t *testing.T) { srv := newNetworkTestServer(t) @@ -660,6 +716,30 @@ func TestDeleteLoadBalancers_ListError_LogsAndSkips(t *testing.T) { } } +// Scenario: HTTP error while verifying LB deletion after polling → ERROR log, no exception +// (unlike other resource types, LB verification failures are non-fatal: Octavia +// listing is known to be slower/less reliable, see PR #261.) +func TestDeleteLoadBalancers_VerificationListError_LogsAndSucceeds(t *testing.T) { + srv := newLBTestServer(t) + lb := lbRecord{ID: "lb-001", Name: lbKubeName("mycluster", "svc")} + srv.lbLists = [][]lbRecord{{lb}} + srv.listStatusOverride = http.StatusInternalServerError + srv.listFailFrom = 2 // pre-delete list (call 1) succeeds; verification (call 2+) fails + + session := srv.authenticate(t) + err := session.DeleteLoadBalancers(context.Background(), logr.Discard(), "mycluster") + + if err != nil { + t.Fatalf("expected nil when verification list fails after deletion, got: %v", err) + } + srv.mu.Lock() + deleted := len(srv.deletedLBs) + srv.mu.Unlock() + if deleted != 1 { + t.Errorf("expected the LB delete to have been attempted, got %d DELETE calls", deleted) + } +} + // ── US3.3: Delete Load Balancers with cascade ───────────────────────────────── // Scenario: Successful deletion with cascade=true From de5091c60f62eda9eb6b545164c5cb2f49439124 Mon Sep 17 00:00:00 2001 From: Mathieu Grzybek Date: Fri, 10 Jul 2026 15:30:33 +0200 Subject: [PATCH 24/27] chores(build): add pre-commit and ignore archives Signed-off-by: Mathieu Grzybek --- .devcontainer/devcontainer.json | 1 - .github/dependabot.yml | 1 - .gitignore | 1 + .pre-commit-config.yaml | 46 +++++++++++++++++++++++++++++++++ hack/boilerplate.go.txt | 2 +- 5 files changed, 48 insertions(+), 3 deletions(-) create mode 100644 .pre-commit-config.yaml diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index a96838b..18224b0 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -32,4 +32,3 @@ "onCreateCommand": "bash .devcontainer/post-install.sh" } - diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 53634b1..5d7e20a 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -1,5 +1,4 @@ version: 2 -updates: updates: # Using dependabot for all GHA violates pinning policy - package-ecosystem: github-actions diff --git a/.gitignore b/.gitignore index af84511..019cce0 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,4 @@ chart/Chart.lock # Nix result result-* +*.tar.gz diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..806c7a3 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,46 @@ +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v5.0.0 + hooks: + - id: trailing-whitespace + - id: end-of-file-fixer + - id: check-yaml + # config/ contains kubebuilder-scaffolded multi-document manifests + # (e.g. config/manager/manager.yaml: Namespace + Deployment). + args: [--allow-multiple-documents] + exclude: ^chart/templates/ + - id: check-added-large-files + - id: check-merge-conflict + + - repo: local + hooks: + - id: go-fmt + name: go fmt + description: Fail if any Go file is not gofmt-formatted (mirrors nix -A tests). + entry: bash -c 'out=$(gofmt -l "$@"); if [ -n "$out" ]; then echo "$out"; exit 1; fi' -- + language: system + types: [go] + + - id: go-vet + name: go vet + description: Run go vet ./... (mirrors nix -A tests / make vet). + entry: bash -c 'go vet ./...' + language: system + types: [go] + pass_filenames: false + + - id: go-test + name: go test + description: Run the unit test suite, excluding e2e (mirrors nix -A tests / make test). + entry: bash -c 'go test $(go list ./... | grep -v /test/e2e)' + language: system + types: [go] + pass_filenames: false + + - id: helm-lint + name: helm lint chart + description: Lint the Helm chart (mirrors the CI helm-lint workflow). + entry: helm lint chart --strict + language: system + files: ^chart/ + pass_filenames: false diff --git a/hack/boilerplate.go.txt b/hack/boilerplate.go.txt index af737e6..70b4dea 100644 --- a/hack/boilerplate.go.txt +++ b/hack/boilerplate.go.txt @@ -12,4 +12,4 @@ distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. -*/ \ No newline at end of file +*/ From eab8f8cdf84f24f4d7448758d6c2786c1114df4d Mon Sep 17 00:00:00 2001 From: Mathieu Grzybek Date: Fri, 10 Jul 2026 15:39:22 +0200 Subject: [PATCH 25/27] chores(ci): run azimuth tests only if the behaviour might change - If a test file has been modified, it means that the behaviour is different. A integration test is then needed. - If the helm chart has been modified, it means that the installation behaviour might change. We need to try to install the product. Signed-off-by: Mathieu Grzybek --- .github/workflows/test-pr.yaml | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-pr.yaml b/.github/workflows/test-pr.yaml index 9dc5b8a..de16282 100644 --- a/.github/workflows/test-pr.yaml +++ b/.github/workflows/test-pr.yaml @@ -31,6 +31,25 @@ jobs: with: ref: ${{ github.event.pull_request.head.sha }} + # Detect which parts of the PR changed, to decide whether the (expensive) + # Azimuth integration test job needs to run. + changes: + runs-on: ubuntu-latest + permissions: + pull-requests: read + outputs: + go_tests: ${{ steps.filter.outputs.go_tests }} + chart: ${{ steps.filter.outputs.chart }} + steps: + - uses: dorny/paths-filter@7b450fff21473bca461d4b92ce414b9d0420d706 # v4.0.2 + id: filter + with: + filters: | + go_tests: + - '**/*_test.go' + chart: + - 'chart/**' + # This job exists so that PRs from outside the main repo are rejected fail_on_remote: needs: [lint] @@ -52,7 +71,8 @@ jobs: security-events: write # required for pushing SARIF files run_azimuth_tests: - needs: [publish_artifacts] + needs: [publish_artifacts, changes] + if: needs.changes.outputs.go_tests == 'true' || needs.changes.outputs.chart == 'true' runs-on: ubuntu-latest steps: # Check out the configuration repository From 159e9d43173dcda9c835b18ed25de05486ebb035 Mon Sep 17 00:00:00 2001 From: Mathieu Grzybek Date: Fri, 10 Jul 2026 16:04:30 +0200 Subject: [PATCH 26/27] chore(build): drop the Dockerfile in favour of Nix, fix dependabot ecosystems The image has been built via `nix-build nix -A image` since the CI rewrite; remove the now-vestigial Dockerfile/.dockerignore and the Makefile docker-build/docker-push/docker-buildx targets, and update every real reference (README, ROADMAP, AGENTS.md, e2e image build/load) to the Nix path. Also fix dependabot.yml, which tracked GitHub Actions and a leftover Python `pip` ecosystem but nothing for go.mod: swap `pip` for `gomod`. Signed-off-by: Mathieu Grzybek --- .dockerignore | 6 ------ .github/dependabot.yml | 16 ++++++++-------- AGENTS.md | 6 ++++-- Dockerfile | 15 --------------- Makefile | 37 ++----------------------------------- README.md | 9 --------- ROADMAP.md | 12 ++++++------ config/manager/manager.yaml | 3 ++- test/e2e/e2e_suite_test.go | 11 +++++++---- test/utils/utils.go | 8 +++++--- 10 files changed, 34 insertions(+), 89 deletions(-) delete mode 100644 .dockerignore delete mode 100644 Dockerfile diff --git a/.dockerignore b/.dockerignore deleted file mode 100644 index d61b0cd..0000000 --- a/.dockerignore +++ /dev/null @@ -1,6 +0,0 @@ -# Exclude the dockerfile itself -Dockerfile - -# Exclude Python caches -**/__pycache__ -*.egg-info diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 5d7e20a..96f4e85 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -18,18 +18,18 @@ updates: - dependency-name: "actions/*" - dependency-name: "github-actions/*" - # Automatically propose PRs for Python dependencies - - package-ecosystem: pip + # Automatically propose PRs for Go module dependencies + - package-ecosystem: gomod directory: "/" schedule: - # Check for new versions daily - interval: daily + # Check for new versions weekly + interval: weekly groups: - pip-updates: + go-updates: patterns: ["*"] labels: - automation - - pip-update + - go-update commit-message: - # Prefix all commit messages with "pip: " - prefix: "pip" + # Prefix all commit messages with "go: " + prefix: "go" diff --git a/AGENTS.md b/AGENTS.md index ab50558..8d76c16 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -158,7 +158,8 @@ make manifests generate # 2. Build & deploy export IMG=/:tag -make docker-build docker-push IMG=$IMG # Or: kind load docker-image $IMG --name +nix-build nix -A image -o image.tar.gz # produces an OCI archive, see nix/default.nix +skopeo copy docker-archive:image.tar.gz "docker://$IMG" # or: kind load image-archive image.tar.gz --name make deploy IMG=$IMG # 3. Test @@ -298,7 +299,8 @@ helm install my-release .//chart/ --namespace --create-namespac ```bash export IMG=/: -make docker-build docker-push IMG=$IMG +nix-build nix -A image -o image.tar.gz # see nix/default.nix +skopeo copy docker-archive:image.tar.gz "docker://$IMG" ``` ## References diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index 3dd2247..0000000 --- a/Dockerfile +++ /dev/null @@ -1,15 +0,0 @@ -FROM golang:1.26 AS builder -ARG TARGETOS -ARG TARGETARCH -WORKDIR /workspace -COPY go.mod go.sum ./ -RUN go mod download -COPY . . -RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} \ - go build -a -o manager ./cmd/main.go - -FROM gcr.io/distroless/static:nonroot -WORKDIR / -COPY --from=builder /workspace/manager . -USER 65532:65532 -ENTRYPOINT ["/manager"] diff --git a/Makefile b/Makefile index 5c25b31..014e246 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,5 @@ -# Image URL to use all building/pushing image targets +# Image URL used to set the manager image in generated/deployed manifests +# (the image itself is built via Nix, see nix/default.nix). IMG ?= controller:latest # YEAR defines the year value used for substituting the YEAR placeholder in the boilerplate header. YEAR ?= $(shell date +%Y) @@ -10,12 +11,6 @@ else GOBIN=$(shell go env GOBIN) endif -# CONTAINER_TOOL defines the container tool to be used for building images. -# Be aware that the target commands are only tested with Docker which is -# scaffolded by default. However, you might want to replace it to use other -# tools. (i.e. podman) -CONTAINER_TOOL ?= docker - # Setting SHELL to bash allows bash commands to be executed by recipes. # Options are set to exit when a recipe line exits non-zero or a piped command fails. SHELL = /usr/bin/env bash -o pipefail @@ -104,34 +99,6 @@ build: manifests generate fmt vet ## Build manager binary. run: manifests generate fmt vet ## Run a controller from your host. go run ./cmd/main.go -# If you wish to build the manager image targeting other platforms you can use the --platform flag. -# (i.e. docker build --platform linux/arm64). However, you must enable docker buildKit for it. -# More info: https://docs.docker.com/develop/develop-images/build_enhancements/ -.PHONY: docker-build -docker-build: ## Build docker image with the manager. - $(CONTAINER_TOOL) build -t ${IMG} . - -.PHONY: docker-push -docker-push: ## Push docker image with the manager. - $(CONTAINER_TOOL) push ${IMG} - -# PLATFORMS defines the target platforms for the manager image be built to provide support to multiple -# architectures. (i.e. make docker-buildx IMG=myregistry/mypoperator:0.0.1). To use this option you need to: -# - be able to use docker buildx. More info: https://docs.docker.com/build/buildx/ -# - have enabled BuildKit. More info: https://docs.docker.com/develop/develop-images/build_enhancements/ -# - be able to push the image to your registry (i.e. if you do not set a valid value via IMG=> then the export will fail) -# To adequately provide solutions that are compatible with multiple platforms, you should consider using this option. -PLATFORMS ?= linux/arm64,linux/amd64,linux/s390x,linux/ppc64le -.PHONY: docker-buildx -docker-buildx: ## Build and push docker image for the manager for cross-platform support - # copy existing Dockerfile and insert --platform=${BUILDPLATFORM} into Dockerfile.cross, and preserve the original Dockerfile - sed -e '1 s/\(^FROM\)/FROM --platform=\$$\{BUILDPLATFORM\}/; t' -e ' 1,// s//FROM --platform=\$$\{BUILDPLATFORM\}/' Dockerfile > Dockerfile.cross - - $(CONTAINER_TOOL) buildx create --name cluster-api-janitor-openstack-builder - $(CONTAINER_TOOL) buildx use cluster-api-janitor-openstack-builder - - $(CONTAINER_TOOL) buildx build --push --platform=$(PLATFORMS) --tag ${IMG} -f Dockerfile.cross . - - $(CONTAINER_TOOL) buildx rm cluster-api-janitor-openstack-builder - rm Dockerfile.cross - .PHONY: build-installer build-installer: manifests generate kustomize ## Generate a consolidated YAML with CRDs and deployment. mkdir -p dist diff --git a/README.md b/README.md index 5de8141..eee45b0 100644 --- a/README.md +++ b/README.md @@ -156,15 +156,6 @@ make build # go build ./cmd/main.go ## Building the OCI image -### Docker (local development) - -```sh -docker build -t cluster-api-janitor-openstack:dev . -``` - -The Dockerfile uses a multi-stage build: `golang:1.26` builder → -`gcr.io/distroless/static:nonroot` runtime (UID 65532, read-only root FS). - ### Nix (reproducible, multi-arch + SBOM) CI uses `nix-build` for reproducible builds. The `tests` derivation runs diff --git a/ROADMAP.md b/ROADMAP.md index 6eaf9ac..92625fd 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -545,15 +545,15 @@ Feature: Configuration via Environment Variables ### Epic 10 — Packaging and Deployment -#### US10.1 — Secure Image (Go Dockerfile) +#### US10.1 — Secure Image (Nix Build) ```gherkin Feature: Secure OCI Image for the Go Operator - Scenario: Multi-stage Go build + Scenario: Reproducible Nix build Given the Go operator source code - When the Dockerfile is built - Then the builder uses golang:1.26 - And the runtime uses gcr.io/distroless/static:nonroot (UID 65532) + When `nix-build nix -A image` is run + Then the manager binary is built with buildGoModule and CGO_ENABLED=0 + And the image contains only the Nix closure required to run the binary Scenario: Image security Given the built image @@ -718,7 +718,7 @@ Feature: Cinder Service Detection with Aliases | Controller | `internal/controller/openstackcluster_controller.go`, `metrics.go` | | Config | `internal/controller/config.go` (env vars) | | Tests | 108 tests (4 packages) | -| Packaging | `Dockerfile`, `nix/default.nix`, `nix/nixpkgs.nix` | +| Packaging | `nix/default.nix`, `nix/nixpkgs.nix` | | Helm | `chart/` — Deployment, ClusterRole, RBAC, health probes | | CI | `.github/workflows/build-push-artifacts.yaml` (Nix + skopeo + SBOM) | diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index fb1d4da..5635130 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -33,7 +33,8 @@ spec: # TODO(user): Uncomment the following code to configure the nodeAffinity expression # according to the platforms which are supported by your solution. # It is considered best practice to support multiple architectures. You can - # build your manager image using the makefile target docker-buildx. + # build a multi-arch manager image with `nix-build nix -A image-arm64` + # (see nix/default.nix). # affinity: # nodeAffinity: # requiredDuringSchedulingIgnoredDuringExecution: diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index 2a843ad..bf935ec 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -23,6 +23,7 @@ import ( "fmt" "os" "os/exec" + "path/filepath" "testing" . "github.com/onsi/ginkgo/v2" @@ -33,7 +34,8 @@ import ( var ( // managerImage is the manager image to be built and loaded for testing. - managerImage = "example.com/cluster-api-janitor-openstack:v0.0.1" + // Must match the name/tag hardcoded in nix/default.nix's `image` derivation. + managerImage = "ghcr.io/azimuth-cloud/cluster-api-janitor-openstack:latest" // shouldCleanupCertManager tracks whether CertManager was installed by this suite. shouldCleanupCertManager = false ) @@ -51,15 +53,16 @@ func TestE2E(t *testing.T) { } var _ = BeforeSuite(func() { - By("building the manager image") - cmd := exec.Command("make", "docker-build", fmt.Sprintf("IMG=%s", managerImage)) + By("building the manager image (Nix)") + imageArchive := filepath.Join(os.TempDir(), "cluster-api-janitor-openstack-e2e-image.tar.gz") + cmd := exec.Command("nix-build", "nix", "-A", "image", "-o", imageArchive) _, err := utils.Run(cmd) ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to build the manager image") // TODO(user): If you want to change the e2e test vendor from Kind, // ensure the image is built and available, then remove the following block. By("loading the manager image on Kind") - err = utils.LoadImageToKindClusterWithName(managerImage) + err = utils.LoadImageArchiveToKindClusterWithName(imageArchive) ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to load the manager image into Kind") configureKubectlKubeRC() diff --git a/test/utils/utils.go b/test/utils/utils.go index a408630..0367c9f 100644 --- a/test/utils/utils.go +++ b/test/utils/utils.go @@ -133,13 +133,15 @@ func IsCertManagerCRDsInstalled() bool { return false } -// LoadImageToKindClusterWithName loads a local docker image to the kind cluster -func LoadImageToKindClusterWithName(name string) error { +// LoadImageArchiveToKindClusterWithName loads an OCI image archive, as +// produced by `nix-build nix -A image`, into the kind cluster. Unlike +// `kind load docker-image`, this does not require a local Docker daemon. +func LoadImageArchiveToKindClusterWithName(archivePath string) error { cluster := defaultKindCluster if v, ok := os.LookupEnv("KIND_CLUSTER"); ok { cluster = v } - kindOptions := []string{"load", "docker-image", name, "--name", cluster} + kindOptions := []string{"load", "image-archive", archivePath, "--name", cluster} kindBinary := defaultKindBinary if v, ok := os.LookupEnv("KIND"); ok { kindBinary = v From 2da1191fbbf4a846cab50a67af4f1ddd9e6cc081 Mon Sep 17 00:00:00 2001 From: Mathieu Grzybek Date: Fri, 10 Jul 2026 16:15:46 +0200 Subject: [PATCH 27/27] chores(build): exclude sbom.cdx.json Signed-off-by: Mathieu Grzybek --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 019cce0..fa018a9 100644 --- a/.gitignore +++ b/.gitignore @@ -10,4 +10,5 @@ chart/Chart.lock # Nix result result-* +sbom.cdx.json *.tar.gz