From 4565b4aa05c9972d1c63b7e8c79590b7bb9ed382 Mon Sep 17 00:00:00 2001 From: ikrispin Date: Mon, 23 Mar 2026 12:25:04 +0200 Subject: [PATCH 1/7] refactor: add OpenShift MCP fallback resilience across all rh-ai-engineer skills --- rh-ai-engineer/README.md | 14 +- .../skills/ai-observability/SKILL.md | 3 + .../skills/debug-inference/SKILL.md | 29 +- .../skills/ds-project-setup/SKILL.md | 63 ++- .../skills/guardrails-config/SKILL.md | 28 +- rh-ai-engineer/skills/model-deploy/SKILL.md | 91 +++- rh-ai-engineer/skills/model-monitor/SKILL.md | 14 +- rh-ai-engineer/skills/model-registry/SKILL.md | 44 +- rh-ai-engineer/skills/nim-setup/SKILL.md | 9 + .../skills/pipeline-manage/SKILL.md | 65 ++- .../skills/references/common-issues.md | 105 ++++- .../openshift-fallback-templates.md | 399 ++++++++++++++++++ .../skills/references/skill-conventions.md | 70 ++- .../skills/serving-runtime-config/SKILL.md | 27 +- .../skills/workbench-manage/SKILL.md | 90 +++- 15 files changed, 927 insertions(+), 124 deletions(-) create mode 100644 rh-ai-engineer/skills/references/openshift-fallback-templates.md diff --git a/rh-ai-engineer/README.md b/rh-ai-engineer/README.md index 62da77dc..7bd84e1f 100644 --- a/rh-ai-engineer/README.md +++ b/rh-ai-engineer/README.md @@ -40,13 +40,15 @@ Automation tools for AI/ML engineers working with Red Hat OpenShift AI (RHOAI). ## MCP Servers -| Server | Type | Required | Description | -|--------|------|----------|-------------| -| `openshift` | Container (podman) | Yes | Kubernetes resource CRUD, pod management, logs, events | -| `rhoai` | Local process (uvx) | Yes | RHOAI-specific operations: model deployment, serving runtimes, data connections, project management | -| `ai-observability` | Remote HTTP | No | vLLM metrics, GPU monitoring, distributed tracing | +| Server | Type | Requirement | Description | +|--------|------|-------------|-------------| +| `openshift` | Container (podman) | **Required** | Kubernetes resource CRUD, pod management, logs, events. The only hard-required server — all RHOAI operations have OpenShift equivalents. | +| `rhoai` | Local process (uvx) | **Preferred** | RHOAI-specific convenience tools: model deployment, serving runtimes, data connections, project management. Automatic fallback to OpenShift when unavailable or returning errors. | +| `ai-observability` | Remote HTTP | **Optional** | vLLM metrics, GPU monitoring, distributed tracing. Skipped when unavailable. | -The `rhoai` MCP server provides high-level, RHOAI-domain-specific tools that simplify model deployment (no YAML construction needed), runtime management (including platform template discovery), and project validation. See [rhoai-mcp](https://github.com/opendatahub-io/rhoai-mcp) for details. +The `openshift` MCP server is the foundation for all skills. It provides reliable Kubernetes resource CRUD operations that serve as automatic fallbacks when RHOAI MCP tools are unavailable or return errors. + +The `rhoai` MCP server provides high-level, RHOAI-domain-specific tools that simplify model deployment (no YAML construction needed), runtime management (including platform template discovery), and project validation. When these tools fail (auth errors, API inconsistencies), skills transparently fall back to equivalent OpenShift operations. See [rhoai-mcp](https://github.com/opendatahub-io/rhoai-mcp) for details. The `ai-observability` MCP server is optional. When available, it enables GPU pre-flight checks before deployment and post-deployment performance validation. See [ai-observability-summarizer](https://github.com/rh-ai-quickstart/ai-observability-summarizer/tree/main/src/mcp_server) for deployment instructions. diff --git a/rh-ai-engineer/skills/ai-observability/SKILL.md b/rh-ai-engineer/skills/ai-observability/SKILL.md index 9011e172..8b8104df 100644 --- a/rh-ai-engineer/skills/ai-observability/SKILL.md +++ b/rh-ai-engineer/skills/ai-observability/SKILL.md @@ -50,10 +50,13 @@ Analyze AI model inference performance, GPU utilization, OpenShift cluster healt - `list_inference_services` - List deployed models with structured status for context - `get_inference_service` - Get InferenceService status for context +**If rhoai is unavailable or returns errors**: Use `openshift` tools as fallback — `resources_list` with `apiVersion: serving.kserve.io/v1beta1`, `kind: InferenceService` replaces `list_inference_services`; `resources_list` with `apiVersion: v1`, `kind: Namespace`, `labelSelector: opendatahub.io/dashboard=true` replaces `list_data_science_projects`. + **Optional MCP Server**: `openshift` ([OpenShift MCP Server](https://github.com/openshift/openshift-mcp-server)) **Optional MCP Tools** (from openshift): - `resources_get` (from openshift) - Get raw resource details for context +- `resources_list` (from openshift) - List InferenceServices, Namespaces (fallback for rhoai tools) - `pods_list` (from openshift) - List predictor pods for correlation context **Common prerequisites** (KUBECONFIG, OpenShift+RHOAI cluster, verification protocol): See [skill-conventions.md](../references/skill-conventions.md). diff --git a/rh-ai-engineer/skills/debug-inference/SKILL.md b/rh-ai-engineer/skills/debug-inference/SKILL.md index 9dcc7ec7..1f16330f 100644 --- a/rh-ai-engineer/skills/debug-inference/SKILL.md +++ b/rh-ai-engineer/skills/debug-inference/SKILL.md @@ -24,21 +24,22 @@ Troubleshoot failed, stuck, or slow InferenceService deployments on Red Hat Open ## Prerequisites -**Required MCP Server**: `rhoai` ([RHOAI MCP Server](https://github.com/opendatahub-io/rhoai-mcp)) +**Required MCP Server**: `openshift` ([OpenShift MCP Server](https://github.com/openshift/openshift-mcp-server)) + +**Required MCP Tools** (from openshift): +- `resources_get` - Get ServingRuntime, NIM Account CR, InferenceService details +- `resources_list` - List InferenceServices (OpenShift fallback) +- `pods_list` - Find predictor/transformer pods +- `pods_log` - Retrieve container logs +- `events_list` - Check events for errors -**Required MCP Tools** (from rhoai): +**Preferred MCP Server**: `rhoai` ([RHOAI MCP Server](https://github.com/opendatahub-io/rhoai-mcp)) — used when available, automatic OpenShift fallback on failure + +**Preferred MCP Tools** (from rhoai): - `list_inference_services` - List deployed models with structured status data - `get_inference_service` - Get detailed deployment status (conditions, endpoint, ready state) - `get_model_endpoint` - Quick check if endpoint is available (early diagnostic) -**Required MCP Server**: `openshift` ([OpenShift MCP Server](https://github.com/openshift/openshift-mcp-server)) - -**Required MCP Tools** (from openshift): -- `resources_get` (from openshift) - Get ServingRuntime, NIM Account CR details -- `pods_list` (from openshift) - Find predictor/transformer pods -- `pods_log` (from openshift) - Retrieve container logs -- `events_list` (from openshift) - Check events for errors - **Optional MCP Server**: `ai-observability` ([AI Observability MCP](https://github.com/rh-ai-quickstart/ai-observability-summarizer)) **Optional MCP Tools** (from ai-observability): @@ -54,6 +55,8 @@ Troubleshoot failed, stuck, or slow InferenceService deployments on Red Hat Open **Common prerequisites** (KUBECONFIG, OpenShift+RHOAI cluster, KServe, verification protocol): See [skill-conventions.md](../references/skill-conventions.md). +**Fallback templates**: See [openshift-fallback-templates.md](../references/openshift-fallback-templates.md) for OpenShift YAML templates used when RHOAI tools are unavailable. + **Additional cluster requirements**: - An existing InferenceService deployment to debug @@ -88,6 +91,8 @@ If user says "list all" or is unsure: - `namespace`: user-specified namespace - REQUIRED - `verbosity`: `"standard"` - OPTIONAL +**If rhoai unavailable or returns error**: Use `resources_list` (from openshift) with `apiVersion: serving.kserve.io/v1beta1`, `kind: InferenceService`, `namespace: [namespace]`. + Present InferenceServices with their status: | Name | Runtime | Ready | URL | Age | @@ -105,11 +110,15 @@ Present InferenceServices with their status: - `namespace`: user-specified namespace - REQUIRED - `verbosity`: `"full"` - REQUIRED +**If rhoai unavailable or returns error**: Use `resources_get` (from openshift) with `apiVersion: serving.kserve.io/v1beta1`, `kind: InferenceService`, `name: [name]`, `namespace: [namespace]`. Extract status from `.status.conditions`. + **Early endpoint check:** **MCP Tool**: `get_model_endpoint` (from rhoai) - `name`: the InferenceService name, `namespace`: user-specified namespace +**If rhoai unavailable or returns error**: Extract endpoint from `.status.url` of the InferenceService obtained via `resources_get` (from openshift). + An empty or error URL indicates deployment issues. Report endpoint availability status. Present status conditions: diff --git a/rh-ai-engineer/skills/ds-project-setup/SKILL.md b/rh-ai-engineer/skills/ds-project-setup/SKILL.md index 1d9212e5..e7ecff0b 100644 --- a/rh-ai-engineer/skills/ds-project-setup/SKILL.md +++ b/rh-ai-engineer/skills/ds-project-setup/SKILL.md @@ -28,26 +28,31 @@ Bootstrap a Red Hat OpenShift AI Data Science Project from scratch. Creates a na ## Prerequisites -**Required MCP Server**: `rhoai` ([RHOAI MCP Server](https://github.com/opendatahub-io/rhoai-mcp)) +**Required MCP Server**: `openshift` ([OpenShift MCP Server](https://github.com/openshift/openshift-mcp-server)) + +**Required MCP Tools** (from openshift): +- `resources_get` - Inspect namespace labels, LimitRange, ResourceQuota, DSPA status +- `resources_list` - List namespaces, Secrets, PVCs (OpenShift fallback for RHOAI tools) +- `resources_create_or_update` - Create namespaces, Secrets, DSPA CRs (OpenShift fallback and primary for pipeline server) -**Required MCP Tools** (from rhoai): +**Preferred MCP Server**: `rhoai` ([RHOAI MCP Server](https://github.com/opendatahub-io/rhoai-mcp)) — used when available, automatic OpenShift fallback on failure + +**Preferred MCP Tools** (from rhoai): - `list_data_science_projects` - List existing RHOAI projects to check for duplicates - `create_data_science_project` - Create namespace with RHOAI labels and dashboard integration -- `get_project_details` - Verify project creation and inspect configuration +- `get_project_details` - Verify project creation and inspect configuration. **Note**: use `name` parameter, not `namespace`. - `get_project_status` - Get comprehensive project status including components - `create_s3_data_connection` - Create S3-compatible data connection secret - `list_data_connections` - List existing data connections in the project - `get_pipeline_server` - Check pipeline server configuration -- `create_pipeline_server` - Configure pipeline server with S3 data connection - `set_model_serving_mode` - Enable single-model or multi-model serving -**Required MCP Server**: `openshift` ([OpenShift MCP Server](https://github.com/openshift/openshift-mcp-server)) - -**Required MCP Tools** (from openshift): -- `resources_get` (from openshift) - Inspect namespace labels, LimitRange, ResourceQuota +Note: `create_pipeline_server` is intentionally excluded — it constructs invalid DSPA manifests. Pipeline server creation always uses OpenShift direct. **Common prerequisites** (KUBECONFIG, OpenShift+RHOAI cluster, verification protocol): See [skill-conventions.md](../references/skill-conventions.md). +**Fallback templates**: See [openshift-fallback-templates.md](../references/openshift-fallback-templates.md) for OpenShift YAML templates used when RHOAI tools are unavailable. + **Additional cluster requirements**: - Cluster admin or namespace creation privileges for the user @@ -82,6 +87,8 @@ Bootstrap a Red Hat OpenShift AI Data Science Project from scratch. Creates a na - If project **exists**: Report to user and offer options: "Project `[name]` already exists. Would you like to: (a) configure additional components on it, or (b) choose a different name?" **WAIT for user decision.** If user chooses (a), skip Step 2 and proceed to optional configuration steps (Steps 3-5). If user chooses (b), repeat the name check. - If project **does not exist**: Continue gathering remaining requirements below. +**If rhoai unavailable or returns error**: Use `resources_list` (from openshift) with `apiVersion: v1`, `kind: Namespace`, `labelSelector: opendatahub.io/dashboard=true`. + **Ask the user for remaining settings:** - **Display name**: Human-readable project name for the RHOAI dashboard - **Description**: Optional project description @@ -111,6 +118,8 @@ Bootstrap a Red Hat OpenShift AI Data Science Project from scratch. Creates a na - `display_name`: human-readable display name - REQUIRED - `description`: project description - OPTIONAL +**If rhoai unavailable or returns error**: Use `resources_create_or_update` (from openshift) to create the Namespace with RHOAI labels. See [openshift-fallback-templates.md](../references/openshift-fallback-templates.md#data-science-project-namespace) for the YAML template. + **Verify creation:** **MCP Tool**: `get_project_details` (from rhoai) @@ -120,6 +129,10 @@ Bootstrap a Red Hat OpenShift AI Data Science Project from scratch. Creates a na Confirm the project was created with proper RHOAI labels (`opendatahub.io/dashboard: "true"`). +**If rhoai unavailable or returns error**: Use `resources_get` (from openshift) with `apiVersion: v1`, `kind: Namespace`, `name: [project-name]`. Check for label `opendatahub.io/dashboard: "true"`. + +**Note**: The `get_project_details` tool requires a `name` parameter (not `namespace`). If the tool returns a parameter error, fall back to OpenShift. + **Error Handling**: - If name already taken -> Offer alternative name or configure existing project - If RBAC error -> Report: "Insufficient permissions to create namespaces. Contact your cluster administrator." @@ -163,6 +176,8 @@ Skip this step if user declined data connections in Step 1. - `secret_key`: secret access key - REQUIRED - `region`: AWS region - OPTIONAL (omit for non-AWS S3) +**If rhoai unavailable or returns error**: Use `resources_create_or_update` (from openshift) to create the Secret with S3 annotations. See [openshift-fallback-templates.md](../references/openshift-fallback-templates.md#s3-data-connection-secret) for the YAML template. + **Verify creation:** **MCP Tool**: `list_data_connections` (from rhoai) @@ -172,6 +187,8 @@ Skip this step if user declined data connections in Step 1. Confirm the data connection appears in the list. +**If rhoai unavailable or returns error**: Use `resources_list` (from openshift) with `apiVersion: v1`, `kind: Secret`, `namespace: [namespace]`, `labelSelector: opendatahub.io/dashboard=true`. Filter results by annotation `opendatahub.io/connection-type: s3`. + **Error Handling**: - If connection name already exists -> Ask: "Data connection `[name]` already exists. Create with a different name?" - If RBAC error -> Report insufficient permissions to create Secrets in namespace @@ -202,14 +219,26 @@ If pipeline server already exists, report its status and ask if user wants to re **WAIT for user to confirm pipeline server setup.** -**MCP Tool**: `create_pipeline_server` (from rhoai) +**Pipeline Server Creation** (OpenShift direct — the `create_pipeline_server` RHOAI tool is not used because it constructs invalid DSPA manifests): -**Parameters**: -- `namespace`: project name - REQUIRED -- `object_storage_secret`: name of the S3 data connection secret - REQUIRED -- `object_storage_bucket`: S3 bucket name (from the data connection) - REQUIRED -- `object_storage_endpoint`: S3 endpoint URL (from the data connection) - REQUIRED -- `object_storage_region`: S3 region - OPTIONAL (default: `"us-east-1"`) +**MCP Tool**: `resources_create_or_update` (from openshift) + +Create a DataSciencePipelinesApplication CR using the template from [openshift-fallback-templates.md](../references/openshift-fallback-templates.md#datasciencepipelinesapplication-dspa). + +**Parameters to fill in the template:** +- `namespace`: target namespace +- `bucket`: S3 bucket name from the data connection +- `host`: S3 endpoint without protocol prefix (e.g., `minio.namespace.svc:9000`) +- `scheme`: `http` or `https` +- `secretName`: name of the S3 data connection secret created in Step 3 +- `region`: AWS region or empty string for MinIO + +**Verify DSPA is ready:** + +**MCP Tool**: `resources_get` (from openshift) +- `apiVersion`: `datasciencepipelinesapplications.opendatahub.io/v1alpha1`, `kind`: `DataSciencePipelinesApplication`, `name`: `dspa`, `namespace`: [namespace] + +Check `.status.conditions` for `Ready=True`. Poll every 15 seconds until ready or timeout (5 minutes). **Verify creation:** @@ -220,6 +249,8 @@ If pipeline server already exists, report its status and ask if user wants to re Confirm the pipeline server is configured and initializing. +**If rhoai unavailable or returns error**: Use `resources_get` (from openshift) for the DSPA CR as described above. + **Error Handling**: - If data connection not found -> Report: "Data connection `[name]` not found in namespace. Create it first." - If pipeline server already exists -> Ask user whether to reconfigure or keep existing @@ -235,6 +266,8 @@ Confirm the pipeline server is configured and initializing. - `namespace`: project name - REQUIRED - `mode`: "single" or "multi" - REQUIRED (default: "single") +**If rhoai unavailable or returns error**: Patch the namespace annotation via `resources_create_or_update` (from openshift). Set annotation `opendatahub.io/model-serving-mode` to `single` or `multi` on the Namespace. + **Final validation:** **MCP Tool**: `get_project_status` (from rhoai) diff --git a/rh-ai-engineer/skills/guardrails-config/SKILL.md b/rh-ai-engineer/skills/guardrails-config/SKILL.md index 9d7ed4cb..29886c79 100644 --- a/rh-ai-engineer/skills/guardrails-config/SKILL.md +++ b/rh-ai-engineer/skills/guardrails-config/SKILL.md @@ -34,9 +34,9 @@ color: blue - `pods_log` (from openshift) - Retrieve orchestrator pod logs for troubleshooting - `events_list` (from openshift) - Check events for deployment issues -**Required MCP Server**: `rhoai` ([RHOAI MCP Server](https://github.com/opendatahub-io/rhoai-mcp)) +**Preferred MCP Server**: `rhoai` ([RHOAI MCP Server](https://github.com/opendatahub-io/rhoai-mcp)) — used when available, automatic OpenShift fallback on failure -**Required MCP Tools** (from rhoai): +**Preferred MCP Tools** (from rhoai): - `list_inference_services` - List deployed models to identify guardrail targets - `get_inference_service` - Get InferenceService details (endpoint, runtime, status) - `get_model_endpoint` - Get the model endpoint URL for orchestrator routing @@ -53,6 +53,8 @@ color: blue **Common prerequisites** (KUBECONFIG, OpenShift+RHOAI cluster, KServe, verification protocol): See [skill-conventions.md](../references/skill-conventions.md). +**Fallback templates**: See [openshift-fallback-templates.md](../references/openshift-fallback-templates.md) for OpenShift YAML templates used when RHOAI tools are unavailable. + **Additional cluster requirements**: - TrustyAI operator installed with guardrails support (RHOAI 2.14+) - At least one deployed LLM InferenceService to guard (via `/model-deploy`) @@ -105,6 +107,8 @@ If user is unsure about target model, use `list_inference_services` (from rhoai) - `namespace`: user-specified namespace - REQUIRED - `verbosity`: `"standard"` - OPTIONAL +**If rhoai unavailable or returns error**: Use `resources_list` (from openshift) with `apiVersion: serving.kserve.io/v1beta1`, `kind: InferenceService`, `namespace: [namespace]`. + Verify the selected InferenceService is Ready: **MCP Tool**: `get_inference_service` (from rhoai) @@ -114,6 +118,8 @@ Verify the selected InferenceService is Ready: - `namespace`: target namespace - REQUIRED - `verbosity`: `"full"` - REQUIRED +**If rhoai unavailable or returns error**: Use `resources_get` (from openshift) with `apiVersion: serving.kserve.io/v1beta1`, `kind: InferenceService`, `name: [name]`, `namespace: [namespace]`. Extract status from `.status.conditions`. + **If not Ready**: Warn user and offer options: (1) Proceed anyway, (2) Invoke `/debug-inference`, (3) Abort. **WAIT for user decision.** **MCP Tool**: `get_model_endpoint` (from rhoai) @@ -122,6 +128,8 @@ Verify the selected InferenceService is Ready: - `name`: selected InferenceService name - REQUIRED - `namespace`: target namespace - REQUIRED +**If rhoai unavailable or returns error**: Extract endpoint from `.status.url` of the InferenceService obtained via `resources_get` (from openshift). + Store the endpoint URL for orchestrator routing. Present configuration summary for confirmation. **WAIT for user to confirm or modify.** ### Step 3: Configure Detectors @@ -138,6 +146,8 @@ Recommended model: `ibm-granite/granite-guardian-3.1-2b` (1 GPU, ~8Gi memory) pe Check if a compatible detector model is already deployed using `list_inference_services` (from rhoai). If one exists, offer to reuse it. **WAIT for user decision.** +**If rhoai unavailable or returns error**: Use `resources_list` (from openshift) with `apiVersion: serving.kserve.io/v1beta1`, `kind: InferenceService`, `namespace: [namespace]` to check for existing detector models. To list available runtimes when `list_serving_runtimes` is unavailable: Use `resources_list` (from openshift) with `apiVersion: serving.kserve.io/v1alpha1`, `kind: ServingRuntime`, `namespace: [namespace]`. + If deploying a new detector: **MCP Tool**: `deploy_model` (from rhoai) @@ -151,6 +161,8 @@ If deploying a new detector: - `gpu_count`: `1` - OPTIONAL - `memory_request`: `"8Gi"` - OPTIONAL +**If rhoai unavailable or returns error**: Use `resources_create_or_update` (from openshift) to create the detector InferenceService CR directly with `apiVersion: serving.kserve.io/v1beta1`, `kind: InferenceService`. + **Ask**: "Deploy the content safety detector model? This creates an additional InferenceService. (yes/no/use-existing)" **WAIT for explicit confirmation.** Monitor deployment until Ready. @@ -228,6 +240,8 @@ First, verify the original model still responds correctly: - `name`: the original InferenceService name - REQUIRED - `namespace`: target namespace - REQUIRED +**If rhoai unavailable or returns error**: Note that `test_model_endpoint` only checks reachability, not actual inference. For a real inference test, use an in-cluster curl command: `curl -X POST [endpoint]/v1/completions -H 'Content-Type: application/json' -d '{"model":"[model]","prompt":"Hello","max_tokens":10}'` + Then test the **guarded endpoint** directly. The guarded endpoint is a different URL from the original — obtain it from the GuardrailsOrchestrator CR status (Step 6). If the guarded endpoint is only available cluster-internally, set up port-forwarding to the orchestrator service first: ``` @@ -269,7 +283,15 @@ For common issues (GPU scheduling, OOMKilled, image pull errors, RBAC), see [com 4. Check detector model pods are running if using model-based detectors 5. Check network policies that might block pod-to-pod communication -### Issue 3: High Latency or False Positives +### Issue 3: GuardrailsOrchestrator RBAC Denied + +**Error**: Cannot create `guardrailsorchestrators` resource — 403 Forbidden + +**Cause**: The user lacks RBAC for the GuardrailsOrchestrator CRD, which is typically cluster-admin only. + +**Solution**: Provide the user with the complete GuardrailsOrchestrator CR YAML and instruct them to ask a cluster administrator to apply it. The detectors ConfigMap (which only requires namespace `edit` role) can still be created by the skill. + +### Issue 4: High Latency or False Positives **Error**: Guarded endpoint is significantly slower than direct endpoint, or legitimate requests are blocked diff --git a/rh-ai-engineer/skills/model-deploy/SKILL.md b/rh-ai-engineer/skills/model-deploy/SKILL.md index 5ad8f308..80247281 100644 --- a/rh-ai-engineer/skills/model-deploy/SKILL.md +++ b/rh-ai-engineer/skills/model-deploy/SKILL.md @@ -24,10 +24,20 @@ Deploy AI/ML models on Red Hat OpenShift AI using KServe. Supports vLLM, NVIDIA ## Prerequisites -**Required MCP Server**: `rhoai` ([RHOAI MCP Server](https://github.com/opendatahub-io/rhoai-mcp)) +**Required MCP Server**: `openshift` ([OpenShift MCP Server](https://github.com/openshift/openshift-mcp-server)) + +**Required MCP Tools** (from openshift): +- `resources_get` - Check NIM Account CR, LimitRange, GPU node taints, InferenceService status +- `resources_list` - Check Knative availability, GPU nodes, existing deployments, ServingRuntimes +- `resources_create_or_update` - Create/patch InferenceService, add tolerations (OpenShift fallback) +- `pods_list` - Check predictor pod status during rollout +- `pods_log` - Retrieve pod logs for debugging +- `events_list` - Check events for errors -**Required MCP Tools** (from rhoai): -- `deploy_model` - Create InferenceService with high-level parameters (no YAML construction needed) +**Preferred MCP Server**: `rhoai` ([RHOAI MCP Server](https://github.com/opendatahub-io/rhoai-mcp)) — used when available, automatic OpenShift fallback on failure + +**Preferred MCP Tools** (from rhoai): +- `deploy_model` - Create InferenceService with high-level parameters (no YAML construction needed). **Known limitation**: does not support tolerations or NIM-specific env vars — see fallback patterns below. - `list_inference_services` - List deployed models with structured status data - `get_inference_service` - Get detailed model deployment status (conditions, endpoint, ready state) - `get_model_endpoint` - Get inference endpoint URL directly @@ -35,15 +45,6 @@ Deploy AI/ML models on Red Hat OpenShift AI using KServe. Supports vLLM, NVIDIA - `list_data_science_projects` - Discover RHOAI projects for namespace validation - `list_data_connections` - Verify model storage access (S3 data connections) -**Required MCP Server**: `openshift` ([OpenShift MCP Server](https://github.com/openshift/openshift-mcp-server)) - -**Required MCP Tools** (from openshift): -- `resources_get` (from openshift) - Check NIM Account CR, LimitRange, GPU node taints -- `resources_list` (from openshift) - Check Knative availability, GPU nodes, existing deployments -- `pods_list` (from openshift) - Check predictor pod status during rollout -- `pods_log` (from openshift) - Retrieve pod logs for debugging -- `events_list` (from openshift) - Check events for errors - **Optional MCP Server**: `ai-observability` ([AI Observability MCP](https://github.com/rh-ai-quickstart/ai-observability-summarizer)) **Optional MCP Tools** (from ai-observability): @@ -53,6 +54,8 @@ Deploy AI/ML models on Red Hat OpenShift AI using KServe. Supports vLLM, NVIDIA **Common prerequisites** (KUBECONFIG, OpenShift+RHOAI cluster, KServe, verification protocol): See [skill-conventions.md](../references/skill-conventions.md). +**Fallback templates**: See [openshift-fallback-templates.md](../references/openshift-fallback-templates.md) for OpenShift YAML templates used when RHOAI tools are unavailable. + **Additional cluster requirements**: - For NIM runtime: NIM platform set up via `/nim-setup` - For vLLM/NIM: NVIDIA GPU nodes available in the cluster @@ -96,6 +99,8 @@ Read [model-deploy-preflight-checklist.md](references/model-deploy-preflight-che - Existing deployments (reference configuration) - Model source accessibility (OCI registry entitlements) +**If rhoai unavailable or returns error**: Use `resources_list` (from openshift) with `apiVersion: v1`, `kind: Namespace`, `labelSelector: opendatahub.io/dashboard=true` to find RHOAI projects. + **Present pre-flight results** in a summary table and note any adjustments made. **WAIT for user confirmation if significant changes were needed** (e.g., deployment mode switch, resource adjustments, tolerations added). ### Step 2: Gather Deployment Details @@ -195,6 +200,8 @@ Offer options: (1) Run `/nim-setup` now, (2) Switch to vLLM, (3) Abort. **WAIT f The response shows existing runtimes and available templates with their supported model formats and `requires_instantiation` flag. +**If rhoai unavailable or returns error**: Use `resources_list` (from openshift) with `apiVersion: serving.kserve.io/v1alpha1`, `kind: ServingRuntime`, `namespace: [target]` to list namespace runtimes, and `kind: ClusterServingRuntime` for platform templates. Filter by label `opendatahub.io/dashboard=true`. + If the needed runtime shows `requires_instantiation: true`, it must first be instantiated via `/serving-runtime-config` or the rhoai `create_serving_runtime` tool. Use the runtime list to select the correct `runtime` name for the deployment. @@ -254,6 +261,38 @@ Use the runtime list to select the correct `runtime` name for the deployment. **Note**: For NIM deployments, ensure the NGC API key secret is referenced. If `deploy_model` does not support NIM-specific env vars, fall back to `resources_create_or_update` (from openshift) with a NIM InferenceService YAML that includes `spec.predictor.env` referencing the `ngc-api-key` secretKeyRef. +#### GPU Toleration Handling + +After `deploy_model` succeeds (or after creating InferenceService via OpenShift fallback), check if GPU tolerations are needed: + +**MCP Tool**: `resources_list` (from openshift) +- `apiVersion`: `v1`, `kind`: `Node`, `labelSelector`: `nvidia.com/gpu.present=true` + +If GPU nodes have taints (check `.spec.taints[]`), patch the InferenceService to add matching tolerations: + +**MCP Tool**: `resources_create_or_update` (from openshift) + +Add tolerations to `spec.predictor.tolerations` matching the discovered taints. Common GPU taints include: +- `nvidia.com/gpu` (Exists/NoSchedule) +- `ai-app=true` (Equal/NoSchedule) +- `ai-node=big` (Equal/NoSchedule) + +After patching, delete the stuck Pending pod to force rescheduling with the new tolerations. + +See [openshift-fallback-templates.md](../references/openshift-fallback-templates.md#toleration-post-deploy-patch) for the complete pattern. + +#### NIM Deployment via OpenShift + +When deploying with NIM runtime and `deploy_model` does not support NIM-specific env vars (NGC_API_KEY secretKeyRef, NIM_MAX_MODEL_LEN, image pull secrets): + +Use `resources_create_or_update` (from openshift) with the NIM InferenceService template from [openshift-fallback-templates.md](../references/openshift-fallback-templates.md#inferenceservice-nim). + +**Key NIM-specific fields:** +- `spec.predictor.containers[0].env` with NGC_API_KEY from secretKeyRef +- `spec.predictor.imagePullSecrets` referencing `ngc-image-pull-secret` +- Use a specific image tag (e.g., `1.8.3`) — the `latest` tag may have CUDA driver incompatibility +- Set `NIM_MAX_MODEL_LEN` to prevent KV cache OOM (use `16384` for T4 GPUs) + **Error Handling**: - If namespace not found -> Report error, suggest creating namespace or using `/ds-project-setup` - If ServingRuntime not found -> Report error, verify runtime name, suggest `/serving-runtime-config` @@ -267,6 +306,8 @@ Poll InferenceService status until ready or timeout (10 minutes). **MCP Tool**: `get_inference_service` (from rhoai) - `name`: deployment name, `namespace`: target namespace, `verbosity`: `"full"` +**If rhoai unavailable or returns error**: Use `resources_get` (from openshift) with `apiVersion: serving.kserve.io/v1beta1`, `kind: InferenceService`, `name: [model-name]`, `namespace: [namespace]`. Check `.status.conditions` for `Ready=True`. + Check the Ready condition and status. Repeat every 15-30 seconds until Ready=True or timeout. **Check predictor pod status:** @@ -285,6 +326,8 @@ Show deployment progress tracking: Pod Scheduled, Image Pulled, Container Starte **MCP Tool**: `get_model_endpoint` (from rhoai) - `name`: deployment name, `namespace`: target namespace +**If rhoai unavailable or returns error**: Extract endpoint from `resources_get` (from openshift) on the InferenceService — the URL is in `.status.url`. + **Report success** showing: model name, runtime, namespace, GPUs, inference endpoint URL, API type (OpenAI-compatible REST), and next steps (`/ai-observability`, `/model-monitor`, `/guardrails-config`). **Provide test commands** based on runtime: @@ -356,6 +399,30 @@ For common issues (GPU scheduling, OOMKilled, image pull errors, RBAC), see [com ``` 3. **Prevention**: Step 1 pre-flight validation now auto-discovers GPU node taints and generates tolerations +### Issue: Pod Stuck Pending Due to GPU Node Taints + +**Error**: Pod shows "0/N nodes are available: node(s) had untolerated taint" in events + +**Cause**: `deploy_model` does not support tolerations. GPU nodes in production clusters are almost always tainted. + +**Solution**: Patch InferenceService with tolerations matching the GPU node taints, then delete the stuck pod. See [common-issues.md](../references/common-issues.md#deploy-model-missing-gpu-tolerations) for details. + +### Issue: NIM CUDA Driver Incompatibility + +**Error**: NIM container crashes with error code 803 or CUDA-related errors + +**Cause**: The `latest` NIM image tag may bundle a CUDA version incompatible with the GPU node's driver. + +**Solution**: Pin NIM image to a specific tag compatible with the cluster's GPU driver version (e.g., `1.8.3` for T4 nodes with older drivers). Check the NVIDIA NIM release notes for driver compatibility. + +### Issue: Stale ReplicaSets After InferenceService Patch + +**Error**: Multiple ReplicaSets exist after patching the InferenceService (e.g., adding tolerations), causing duplicate Pending pods + +**Cause**: Each InferenceService spec change triggers a new ReplicaSet. Old ReplicaSets are not automatically cleaned up. + +**Solution**: Scale down stale ReplicaSets to 0 replicas via `resources_create_or_update` (from openshift), or delete them. Identify the current ReplicaSet by checking which one has the latest creation timestamp. + ## Dependencies ### MCP Tools diff --git a/rh-ai-engineer/skills/model-monitor/SKILL.md b/rh-ai-engineer/skills/model-monitor/SKILL.md index 9e7244d0..2ecf5a7b 100644 --- a/rh-ai-engineer/skills/model-monitor/SKILL.md +++ b/rh-ai-engineer/skills/model-monitor/SKILL.md @@ -34,9 +34,9 @@ color: blue - `events_list` (from openshift) - Check events for TrustyAI deployment issues - `prometheus_query` (from openshift) - Query TrustyAI metrics (trustyai_spd, trustyai_dir, drift metrics) -**Required MCP Server**: `rhoai` ([RHOAI MCP Server](https://github.com/opendatahub-io/rhoai-mcp)) +**Preferred MCP Server**: `rhoai` ([RHOAI MCP Server](https://github.com/opendatahub-io/rhoai-mcp)) — used when available, automatic OpenShift fallback on failure -**Required MCP Tools** (from rhoai): +**Preferred MCP Tools** (from rhoai): - `list_inference_services` - List deployed models to identify monitoring targets - `get_inference_service` - Get InferenceService details (model format, runtime, status) - `list_data_science_projects` - Validate namespace is an RHOAI Data Science Project @@ -48,6 +48,8 @@ color: blue **Common prerequisites** (KUBECONFIG, OpenShift+RHOAI cluster, KServe, verification protocol): See [skill-conventions.md](../references/skill-conventions.md). +**Fallback templates**: See [openshift-fallback-templates.md](../references/openshift-fallback-templates.md) for OpenShift YAML templates used when RHOAI tools are unavailable. + **Additional cluster requirements**: - TrustyAI operator enabled in the DataScienceCluster CR - At least one deployed InferenceService to monitor (via `/model-deploy`) @@ -99,6 +101,14 @@ If user is unsure about target model, use `list_inference_services` (from rhoai) - `namespace`: user-specified namespace - REQUIRED - `verbosity`: `"standard"` - OPTIONAL +**If rhoai unavailable or returns error**: Use `resources_list` (from openshift) with `apiVersion: serving.kserve.io/v1beta1`, `kind: InferenceService`, `namespace: [namespace]`. + +To validate namespace is a Data Science Project when `list_data_science_projects` (from rhoai) is unavailable: Use `resources_list` (from openshift) with `apiVersion: v1`, `kind: Namespace`, `labelSelector: opendatahub.io/dashboard=true`. + +To get InferenceService details when `get_inference_service` (from rhoai) is unavailable: Use `resources_get` (from openshift) with `apiVersion: serving.kserve.io/v1beta1`, `kind: InferenceService`, `name: [name]`, `namespace: [namespace]`. Extract status from `.status.conditions`. + +**Important**: TrustyAI payload logging requires **Knative/Serverless** deployment mode. In RawDeployment mode, inference data does not reach TrustyAI for bias/drift analysis. If the InferenceService uses `serving.kserve.io/deploymentMode: RawDeployment`, warn the user that payload logging will not work and suggest switching to Serverless mode if Knative is available. + Present configuration summary for confirmation. **WAIT for user to confirm or modify.** ### Step 3: Check/Create TrustyAIService in Namespace diff --git a/rh-ai-engineer/skills/model-registry/SKILL.md b/rh-ai-engineer/skills/model-registry/SKILL.md index ba568388..4584dc1b 100644 --- a/rh-ai-engineer/skills/model-registry/SKILL.md +++ b/rh-ai-engineer/skills/model-registry/SKILL.md @@ -24,28 +24,34 @@ Register, version, and manage ML models in the Red Hat OpenShift AI Model Regist ## Prerequisites -**Required MCP Server**: `rhoai` ([RHOAI MCP Server](https://github.com/opendatahub-io/rhoai-mcp)) +**Required MCP Server**: `openshift` ([OpenShift MCP Server](https://github.com/openshift/openshift-mcp-server)) + +**Required MCP Tools** (from openshift): +- `resources_create_or_update` (from openshift) - Create RegisteredModel, ModelVersion, and ModelArtifact resources +- `resources_get` (from openshift) - Inspect Model Registry instance and CRs +- `resources_list` (from openshift) - List Model Registry instances and resources -**Required MCP Tools** (from rhoai): +**Preferred MCP Server**: `rhoai` ([RHOAI MCP Server](https://github.com/opendatahub-io/rhoai-mcp)) — used when available, automatic OpenShift fallback on failure + +**Preferred MCP Tools** (from rhoai): - `list_registered_models` - List registered models with pagination, auto-detects Registry vs Catalog - `get_registered_model` - Get model details by ID, optionally with all versions - `list_model_versions` - List versions of a registered model with pagination - `get_model_version` - Get specific version details (state, author, custom properties) - `get_model_artifacts` - Get artifacts (storage URIs) for a model version - `get_model_benchmarks` - Get benchmark data (latency, throughput, GPU memory) -- `list_catalog_sources` - List Model Catalog source categories - `get_catalog_model_artifacts` - Get artifacts from Model Catalog entries - `list_data_science_projects` - Validate namespace is an RHOAI Data Science Project - `list_data_connections` - Verify S3 data connections exist in target namespace (for promotion) -**Required MCP Server**: `openshift` ([OpenShift MCP Server](https://github.com/openshift/openshift-mcp-server)) +**Common prerequisites** (KUBECONFIG, OpenShift+RHOAI cluster, verification protocol): See [skill-conventions.md](../references/skill-conventions.md). -**Required MCP Tools** (from openshift): -- `resources_create_or_update` (from openshift) - Create RegisteredModel, ModelVersion, and ModelArtifact resources -- `resources_get` (from openshift) - Inspect Model Registry instance and CRs -- `resources_list` (from openshift) - List Model Registry instances and resources +**Fallback templates**: See [openshift-fallback-templates.md](../references/openshift-fallback-templates.md) for OpenShift YAML templates used when RHOAI tools are unavailable. -**Common prerequisites** (KUBECONFIG, OpenShift+RHOAI cluster, verification protocol): See [skill-conventions.md](../references/skill-conventions.md). +**Important**: Model Registry RHOAI tools may fail with DNS/connection errors because the RHOAI MCP server runs outside the cluster and cannot resolve internal service DNS names. If this happens: +1. Check if an external Route exists: `resources_list` (from openshift) Routes in the model registry namespace +2. If no Route: set up port-forwarding — `oc port-forward svc/modelregistry-sample 8085:8085 -n rhoai-model-registries` +3. For registry CRUD: use `resources_create_or_update` / `resources_get` / `resources_list` via OpenShift MCP for RegisteredModel, ModelVersion, and ModelArtifact CRs **Additional cluster requirements**: - Model Registry operator installed and a ModelRegistry instance deployed in the cluster @@ -76,11 +82,13 @@ Ask the user what they want to do: **Browse** catalog, **List** models, **View** Ask for the target namespace (required except for catalog browsing). Validate via `list_data_science_projects` (from rhoai). If invalid, suggest `/ds-project-setup`. +**If rhoai unavailable or returns error**: Use `resources_list` (from openshift) with `apiVersion: v1`, `kind: Namespace`, `labelSelector: opendatahub.io/dashboard=true` to validate namespace is a Data Science Project. + Route: Browse/List -> Step 2, View -> Step 3, Register -> Step 4, Version -> Step 5, Promote -> Step 6, Deploy -> Step 7. ### Step 2: Browse Model Catalog / List Registered Models -For catalog browsing, first use `list_catalog_sources` (from rhoai) to show available sources. +For catalog browsing, use `resources_list` (from openshift) with the appropriate catalog source CRD to show available sources. **MCP Tool**: `list_registered_models` (from rhoai) @@ -89,6 +97,8 @@ For catalog browsing, first use `list_catalog_sources` (from rhoai) to show avai - `limit`: number of models to return - OPTIONAL - `verbosity`: `"standard"` or `"minimal"` - OPTIONAL +**If rhoai unavailable or returns error**: Use `resources_list` (from openshift) with `apiVersion: modelregistry.opendatahub.io/v1alpha1`, `kind: RegisteredModel`. + For catalog model artifacts, use `get_catalog_model_artifacts` (from rhoai) with `model_name` (REQUIRED). **Error Handling**: @@ -98,8 +108,12 @@ For catalog model artifacts, use `get_catalog_model_artifacts` (from rhoai) with Use `get_registered_model` (from rhoai) with `model_id` and `include_versions=true` to get model details with version summary. +**If rhoai unavailable or returns error**: Use `resources_get` (from openshift) with `apiVersion: modelregistry.opendatahub.io/v1alpha1`, `kind: RegisteredModel`, `name: [name]`, `namespace: [namespace]`. + For version listing, use `list_model_versions` (from rhoai) with `model_id` (REQUIRED). +**If rhoai unavailable or returns error**: Use `resources_list` (from openshift) with `apiVersion: modelregistry.opendatahub.io/v1alpha1`, `kind: ModelVersion`. + For specific version details: `get_model_version` (from rhoai) with `version_id` (REQUIRED). For artifacts (storage URIs): `get_model_artifacts` (from rhoai) with `version_id` (REQUIRED). @@ -184,7 +198,15 @@ Delegate to `/model-deploy` with the extracted storage URI and model format. **Cause**: Model Registry operator not installed or no ModelRegistry instance created. **Solution**: Check via `resources_list` (from openshift) for `ModelRegistry` CRs. If missing, install via OperatorHub. -### Issue 2: Artifact Storage Inaccessible During Promotion +### Issue 2: Model Registry Unreachable from MCP + +**Error**: RHOAI MCP tools for model registry return connection errors or 404 + +**Cause**: The RHOAI MCP server runs outside the cluster and cannot resolve cluster-internal DNS. External routes may also be behind an OAuth proxy. + +**Solution**: See [common-issues.md](../references/common-issues.md#model-registry-internal-dns-unreachable) for port-forwarding and Route-based solutions. + +### Issue 3: Artifact Storage Inaccessible During Promotion **Cause**: PVC-based storage is namespace-local; S3 credentials may not exist in the target namespace. **Solution**: For S3, verify data connection exists in target namespace via `list_data_connections`. For PVCs, recommend migrating to S3 for cross-namespace portability. diff --git a/rh-ai-engineer/skills/nim-setup/SKILL.md b/rh-ai-engineer/skills/nim-setup/SKILL.md index 25d485e6..d72b07b5 100644 --- a/rh-ai-engineer/skills/nim-setup/SKILL.md +++ b/rh-ai-engineer/skills/nim-setup/SKILL.md @@ -351,6 +351,15 @@ See [Prerequisites](#prerequisites) for the complete list of required and option - `/model-deploy` - Deploy a model using NIM runtime after setup is complete - `/serving-runtime-config` - Configure custom serving runtimes if NIM doesn't fit +### NIM Deployment Handoff to /model-deploy + +When handing off to `/model-deploy` after NIM setup, note these NIM-specific considerations: + +- **NIM URI scheme**: NIM models may use a `nim://` URI scheme which the `deploy_model` RHOAI tool may not recognize. If this happens, `/model-deploy` will fall back to creating the InferenceService via OpenShift direct with the NIM container image, NGC credentials, and NIM-specific env vars. +- **CUDA driver compatibility**: The `latest` NIM image tag may bundle a CUDA version incompatible with the cluster's GPU drivers. Always recommend a specific tag (e.g., `1.8.3`) matched to the GPU driver version. +- **NIM_MAX_MODEL_LEN**: NIM defaults to a very large context length that can cause KV cache OOM on smaller GPUs. Recommend setting `NIM_MAX_MODEL_LEN=16384` for T4/A10 GPUs. +- **GPU tolerations**: GPU nodes are almost always tainted in production. `/model-deploy` will automatically detect and add tolerations after deployment. + ### Reference Documentation - [supported-runtimes.md](../../docs/references/supported-runtimes.md) - NIM runtime capabilities and requirements - [live-doc-lookup.md](../references/live-doc-lookup.md) - Protocol for fetching current RHOAI/NIM documentation diff --git a/rh-ai-engineer/skills/pipeline-manage/SKILL.md b/rh-ai-engineer/skills/pipeline-manage/SKILL.md index 59c40dfb..0eb08d73 100644 --- a/rh-ai-engineer/skills/pipeline-manage/SKILL.md +++ b/rh-ai-engineer/skills/pipeline-manage/SKILL.md @@ -26,12 +26,22 @@ Create, run, schedule, and monitor Data Science Pipelines (Kubeflow Pipelines 2. ## Prerequisites -**Required MCP Server**: `rhoai` ([RHOAI MCP Server](https://github.com/opendatahub-io/rhoai-mcp)) +**Required MCP Server**: `openshift` ([OpenShift MCP Server](https://github.com/openshift/openshift-mcp-server)) + +**Required MCP Tools** (from openshift): +- `resources_create_or_update` - Create DSPA CR (pipeline server), PipelineRun and ScheduledWorkflow CRs +- `resources_list` - List PipelineRun resources, DSPA status +- `resources_get` - Get PipelineRun status, DSPA details +- `resources_delete` - Delete pipeline run resources, DSPA +- `events_list` - Check pipeline pod events for errors +- `pods_list` - List pipeline step pods +- `pods_log` - Retrieve pipeline step container logs -**Required MCP Tools** (from rhoai): +**Preferred MCP Server**: `rhoai` ([RHOAI MCP Server](https://github.com/opendatahub-io/rhoai-mcp)) — used when available, automatic OpenShift fallback on failure + +**Preferred MCP Tools** (from rhoai): - `list_data_science_projects` - Validate namespace is an RHOAI Data Science Project - `get_pipeline_server` - Check pipeline server (DSPA) status in a project -- `create_pipeline_server` - Create DSPA with S3 storage configuration - `delete_pipeline_server` - Delete pipeline server and all pipeline infrastructure - `list_resources` - List pipeline resources in a namespace (resource_type="pipelines") - `get_resource` - Get pipeline resource details (resource_type="pipeline") @@ -41,19 +51,10 @@ Create, run, schedule, and monitor Data Science Pipelines (Kubeflow Pipelines 2. - `list_data_connections` - Verify S3 data connections for pipeline artifact storage - `project_summary` - Project overview including pipeline status -**Required MCP Server**: `openshift` ([OpenShift MCP Server](https://github.com/openshift/openshift-mcp-server)) - -**Required MCP Tools** (from openshift): -- `resources_create_or_update` (from openshift) - Create PipelineRun and ScheduledWorkflow CRs -- `resources_list` (from openshift) - List PipelineRun resources by apiVersion/kind -- `resources_get` (from openshift) - Get PipelineRun status and task details -- `resources_delete` (from openshift) - Delete pipeline run resources -- `events_list` (from openshift) - Check pipeline pod events for errors -- `pods_list` (from openshift) - List pipeline step pods -- `pods_log` (from openshift) - Retrieve pipeline step container logs - **Common prerequisites** (KUBECONFIG, OpenShift+RHOAI cluster, verification protocol): See [skill-conventions.md](../references/skill-conventions.md). +**Fallback templates**: See [openshift-fallback-templates.md](../references/openshift-fallback-templates.md) for OpenShift YAML templates used when RHOAI tools are unavailable. + **Additional cluster requirements**: - Target namespace is an RHOAI Data Science Project (label: `opendatahub.io/dashboard: "true"`) - Pipeline server requires S3-compatible object storage (a data connection must exist in the project) @@ -84,26 +85,42 @@ Ask the user what they want to do: **Setup** server, **List** pipelines/runs, ** Ask for target namespace. Validate via `list_data_science_projects` (from rhoai). If invalid, suggest `/ds-project-setup`. +**If rhoai unavailable or returns error**: Use `resources_list` (from openshift) with `apiVersion: v1`, `kind: Namespace`, `labelSelector: opendatahub.io/dashboard=true`. + Route: Setup -> Step 2, List -> Step 3, Run -> Step 4, Schedule -> Step 5, Monitor -> Step 6, Logs -> Step 7, Delete -> Step 8. ### Step 2: Verify / Setup Pipeline Server Check via `get_pipeline_server` (from rhoai) with `namespace`. If healthy, proceed. If unhealthy, offer diagnostics via `diagnose_resource`. If not exists, offer setup. +**If rhoai unavailable or returns error**: Use `resources_get` (from openshift) with `apiVersion: datasciencepipelinesapplications.opendatahub.io/v1alpha1`, `kind: DataSciencePipelinesApplication`, `name: dspa`, `namespace: [namespace]`. Check `.status.conditions` for `Ready=True`. + **For setup**: Check data connections via `list_data_connections` (from rhoai). If none exist, offer to delegate to `/ds-project-setup`. +**If rhoai unavailable or returns error**: Use `resources_list` (from openshift) with `apiVersion: v1`, `kind: Secret`, `namespace: [namespace]`, `labelSelector: opendatahub.io/dashboard=true`. Filter by annotation `opendatahub.io/connection-type: s3`. + **Gather:** Select from available data connections (the data connection name is the S3 secret name). The bucket, endpoint, and region can be extracted from the data connection secret. Present configuration for review. **WAIT for confirmation.** -**MCP Tool**: `create_pipeline_server` (from rhoai) +**Pipeline Server Creation** (OpenShift direct — `create_pipeline_server` from rhoai is not used because it constructs invalid DSPA manifests): -**Parameters**: -- `namespace`: target namespace - REQUIRED -- `object_storage_secret`: S3 credentials secret name - REQUIRED -- `object_storage_bucket`: S3 bucket name - REQUIRED -- `object_storage_endpoint`: S3 endpoint URL - REQUIRED -- `object_storage_region`: S3 region - OPTIONAL (default: `"us-east-1"`) +**MCP Tool**: `resources_create_or_update` (from openshift) + +Create a DataSciencePipelinesApplication CR. See [openshift-fallback-templates.md](../references/openshift-fallback-templates.md#datasciencepipelinesapplication-dspa) for the YAML template. -Poll `get_pipeline_server` until ready or timeout. +**Parameters to fill in the template:** +- `namespace`: target namespace +- `bucket`: S3 bucket name from the data connection +- `host`: S3 endpoint without protocol prefix (e.g., `minio.namespace.svc:9000`) +- `scheme`: `http` or `https` +- `secretName`: name of the S3 data connection secret +- `region`: AWS region or empty string for MinIO + +**Verify DSPA is ready:** + +**MCP Tool**: `resources_get` (from openshift) +- `apiVersion`: `datasciencepipelinesapplications.opendatahub.io/v1alpha1`, `kind`: `DataSciencePipelinesApplication`, `name`: `dspa`, `namespace`: [namespace] + +Check `.status.conditions` for `Ready=True`. Poll every 15 seconds until ready or timeout (5 minutes). **Error Handling**: - If S3 secret not found -> Suggest creating via `/ds-project-setup` @@ -117,6 +134,8 @@ For specific run status: `resource_status` (from rhoai) with `resource_type="pip For project-wide overview: `project_summary` (from rhoai) with `namespace`. +**If rhoai unavailable or returns error**: Use `resources_list` (from openshift) with `apiVersion: tekton.dev/v1`, `kind: PipelineRun`, `namespace: [namespace]` to list pipeline runs directly. + If pipeline server not configured, suggest setup via Step 2. ### Step 4: Submit a Pipeline Run @@ -158,6 +177,8 @@ Convert natural language to cron if needed. Present schedule configuration for r **For deeper diagnostics**: `diagnose_resource` (from rhoai) with `resource_type="pipeline"`, `name`, `namespace`. +**If rhoai unavailable or returns error**: Use `resources_get` (from openshift) with `apiVersion: tekton.dev/v1`, `kind: PipelineRun`, `name: [run-name]`, `namespace: [namespace]`. Extract task status from `.status.childReferences` or `.status.taskRuns`. + **Track step-level progress** via `resources_get` (from openshift) with apiVersion `tekton.dev/v1`, kind `PipelineRun`. Extract task statuses from `.status.childReferences` or `.status.taskRuns`. **List pipeline pods** via `pods_list` (from openshift) with `namespace` and `labelSelector="tekton.dev/pipelineRun="`. diff --git a/rh-ai-engineer/skills/references/common-issues.md b/rh-ai-engineer/skills/references/common-issues.md index 3e454dc4..20f375ca 100644 --- a/rh-ai-engineer/skills/references/common-issues.md +++ b/rh-ai-engineer/skills/references/common-issues.md @@ -1,10 +1,10 @@ --- title: Common Issues Across Skills category: references -tags: [troubleshooting, gpu, oom, image-pull, rbac, common-issues] -semantic_keywords: [GPU scheduling failure, OOMKilled, image pull error, RBAC permissions, common deployment errors] -use_cases: [model-deploy, debug-inference, workbench-manage, nim-setup, serving-runtime-config] -last_updated: 2026-03-12 +tags: [troubleshooting, gpu, oom, image-pull, rbac, common-issues, rhoai-mcp, tolerations, notebook-images, pipeline-server] +semantic_keywords: [GPU scheduling failure, OOMKilled, image pull error, RBAC permissions, common deployment errors, RHOAI MCP authentication, notebook image mismatch, model registry DNS, GPU tolerations, dangerous operations disabled] +use_cases: [model-deploy, debug-inference, workbench-manage, nim-setup, serving-runtime-config, ds-project-setup, pipeline-manage, model-registry, guardrails-config, model-monitor] +last_updated: 2026-03-23 --- # Common Issues Across Skills @@ -82,3 +82,100 @@ Shared troubleshooting patterns that apply to multiple rh-ai-engineer skills. In 2. Identify the required role: which API group, resource, and verb is needed 3. Suggest contacting the cluster administrator to grant the necessary RoleBinding or ClusterRoleBinding 4. For namespace-scoped operations: verify the user has at least `edit` role in the target namespace + +## RHOAI MCP Authentication Failures + +**Applies to**: All skills that use RHOAI MCP tools + +**Error**: RHOAI MCP tools return "Unauthorized" (HTTP 401/403) for most operations + +**Cause**: The RHOAI MCP server may use a different authentication mechanism than the user's `oc` token, or the token may have expired. This was observed to cause a 78% failure rate across RHOAI tool calls in testing. + +**Solution:** +1. Skills automatically fall back to OpenShift MCP tools when RHOAI tools fail — no user action needed +2. If you want to restore RHOAI MCP functionality: + - Re-login with `oc login` to refresh the token + - Verify `RHOAI_MCP_KUBECONFIG_PATH` points to a valid, current kubeconfig + - Restart Claude Code to reload MCP server connections +3. All RHOAI operations have OpenShift equivalents — see [openshift-fallback-templates.md](openshift-fallback-templates.md) + +## RHOAI MCP API Inconsistencies + +**Applies to**: `/ds-project-setup`, `/workbench-manage`, `/pipeline-manage` + +**Error**: Various RHOAI MCP tool failures: +- `get_project_details` requires `name` parameter but agent may pass `namespace` +- `manage_resource(action=stop)` returns "Unsupported Media Type" +- `create_pipeline_server` returns "Unprocessable Entity" (invalid DSPA manifest) + +**Cause**: RHOAI MCP tool parameter naming differs from Kubernetes conventions, and some tools construct invalid resource manifests. + +**Solution:** +1. `get_project_details`: Always use the `name` parameter (not `namespace`). On error, fall back to `resources_get` for the Namespace. +2. `manage_resource(stop/start)`: Use the annotation-patch fallback — patch the Notebook CR annotation `kubeflow-resource-stopped` via OpenShift MCP. +3. `create_pipeline_server`: **Do not use this tool.** Always create DSPA CRs directly via `resources_create_or_update` with the correct YAML template from [openshift-fallback-templates.md](openshift-fallback-templates.md#datasciencepipelinesapplication-dspa). + +## Notebook Image Names Mismatch + +**Applies to**: `/workbench-manage` + +**Error**: `list_notebook_images` (from rhoai) returns image names like `jupyter-pytorch-notebook:2024.1` that don't exist in the internal registry. The actual ImageStream is named `pytorch` (not `jupyter-pytorch-notebook`). + +**Cause**: The RHOAI MCP tool returns hardcoded display names instead of the actual ImageStream names from the cluster. + +**Solution:** +- **Do not use `list_notebook_images`.** This tool is permanently replaced. +- Use the ImageStream lookup pattern via OpenShift MCP: + 1. `resources_list` ImageStreams in `redhat-ods-applications` namespace with label `opendatahub.io/notebook-image=true` + 2. `resources_get` each ImageStream to extract actual tag names and image references +- See [openshift-fallback-templates.md](openshift-fallback-templates.md#notebook-image-discovery-imagestream-lookup) for the complete pattern. + +## Model Registry Internal DNS Unreachable + +**Applies to**: `/model-registry` + +**Error**: `list_registered_models` and other Model Registry RHOAI tools fail with connection errors or return 404 + +**Cause**: The RHOAI MCP server runs outside the cluster and cannot resolve cluster-internal DNS names (e.g., `modelregistry-sample.rhoai-model-registries.svc.cluster.local`). Additionally, external routes may be behind an OAuth proxy. + +**Solution:** +1. Check if an external Route exists for the model registry: `resources_list` Routes in the model registry namespace +2. If no Route exists, set up port-forwarding: `oc port-forward svc/modelregistry-sample 8085:8085 -n rhoai-model-registries` +3. If an OAuth proxy blocks the Route, obtain a token via `oc whoami -t` and pass it as a Bearer token +4. For registry CRUD operations, use `resources_create_or_update` / `resources_get` / `resources_list` via OpenShift MCP for RegisteredModel, ModelVersion, and ModelArtifact CRs + +## Deploy Model Missing GPU Tolerations + +**Applies to**: `/model-deploy`, `/debug-inference` + +**Error**: Model pod stuck in Pending state after `deploy_model` succeeds. Events show "0/N nodes are available: ... node(s) had untolerated taint" + +**Cause**: The `deploy_model` RHOAI tool does not accept toleration or nodeSelector parameters. In production clusters, GPU nodes are almost always tainted (common taints: `nvidia.com/gpu`, `ai-app=true`, `ai-node=big`). + +**Solution:** +1. After `deploy_model` completes, check GPU node taints: `resources_list` Nodes with label `nvidia.com/gpu.present=true` +2. If taints are found, patch the InferenceService to add tolerations via `resources_create_or_update`: + ```yaml + spec: + predictor: + tolerations: + - key: "nvidia.com/gpu" + operator: "Exists" + effect: "NoSchedule" + # Add cluster-specific taints discovered from node inspection + ``` +3. Delete the stuck pod to force rescheduling with new tolerations +4. See [openshift-fallback-templates.md](openshift-fallback-templates.md#toleration-post-deploy-patch) for the complete pattern + +## Dangerous Operations Disabled in RHOAI MCP + +**Applies to**: `/workbench-manage` + +**Error**: `delete_workbench` and `manage_resource(action=delete)` return "Dangerous operations are disabled" + +**Cause**: The RHOAI MCP server has a safety flag that blocks all destructive operations by default. + +**Solution:** +1. Use `resources_delete` from the OpenShift MCP server instead +2. Always apply human-in-the-loop confirmation before deleting (display resource name, warn about data loss) +3. For workbench deletion, warn that associated PVC data may be lost if the PVC is also deleted diff --git a/rh-ai-engineer/skills/references/openshift-fallback-templates.md b/rh-ai-engineer/skills/references/openshift-fallback-templates.md new file mode 100644 index 00000000..1932963b --- /dev/null +++ b/rh-ai-engineer/skills/references/openshift-fallback-templates.md @@ -0,0 +1,399 @@ +--- +title: OpenShift Fallback Templates +category: references +tags: [openshift, fallback, yaml-templates, resilience] +semantic_keywords: [OpenShift MCP fallback, RHOAI alternative, direct Kubernetes API, resource creation templates] +use_cases: [ds-project-setup, model-deploy, workbench-manage, pipeline-manage, serving-runtime-config, model-registry] +last_updated: 2026-03-23 +--- + +# OpenShift Fallback Templates + +YAML templates for creating RHOAI resources directly via the OpenShift MCP server (`resources_create_or_update`). Use these when the RHOAI MCP tool for the operation is unavailable or returns an error. + +All templates use placeholder values in `[BRACKETS]` that must be replaced with actual values. + +## Data Science Project (Namespace) + +**Replaces**: `create_data_science_project` (from rhoai) + +**MCP Tool**: `resources_create_or_update` (from openshift) + +```yaml +apiVersion: v1 +kind: Namespace +metadata: + name: [PROJECT_NAME] + labels: + opendatahub.io/dashboard: "true" + kubernetes.io/metadata.name: [PROJECT_NAME] + annotations: + openshift.io/display-name: [DISPLAY_NAME] + openshift.io/description: [DESCRIPTION] +``` + +**Listing projects** (replaces `list_data_science_projects`): + +**MCP Tool**: `resources_list` (from openshift) +- `apiVersion`: `v1`, `kind`: `Namespace`, `labelSelector`: `opendatahub.io/dashboard=true` + +## S3 Data Connection Secret + +**Replaces**: `create_s3_data_connection` (from rhoai) + +**MCP Tool**: `resources_create_or_update` (from openshift) + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: [CONNECTION_NAME] + namespace: [NAMESPACE] + labels: + opendatahub.io/dashboard: "true" + opendatahub.io/managed: "true" + annotations: + opendatahub.io/connection-type: s3 + openshift.io/display-name: [DISPLAY_NAME] +type: Opaque +stringData: + AWS_ACCESS_KEY_ID: [ACCESS_KEY] + AWS_SECRET_ACCESS_KEY: [SECRET_KEY] + AWS_S3_BUCKET: [BUCKET_NAME] + AWS_S3_ENDPOINT: [ENDPOINT_URL] + AWS_DEFAULT_REGION: [REGION] +``` + +**Listing data connections** (replaces `list_data_connections`): + +**MCP Tool**: `resources_list` (from openshift) +- `apiVersion`: `v1`, `kind`: `Secret`, `namespace`: `[NAMESPACE]`, `labelSelector`: `opendatahub.io/dashboard=true` +- Filter results by annotation `opendatahub.io/connection-type: s3` + +## InferenceService (vLLM) + +**Replaces**: `deploy_model` (from rhoai) for vLLM deployments + +**MCP Tool**: `resources_create_or_update` (from openshift) + +```yaml +apiVersion: serving.kserve.io/v1beta1 +kind: InferenceService +metadata: + name: [MODEL_NAME] + namespace: [NAMESPACE] + labels: + opendatahub.io/dashboard: "true" + annotations: + serving.kserve.io/deploymentMode: RawDeployment + openshift.io/display-name: [DISPLAY_NAME] +spec: + predictor: + model: + modelFormat: + name: vLLM + runtime: [SERVING_RUNTIME_NAME] + storageUri: "hf://[HF_MODEL_ID]" + resources: + limits: + cpu: "[CPU_LIMIT]" + memory: "[MEMORY_LIMIT]" + nvidia.com/gpu: "[GPU_COUNT]" + requests: + cpu: "[CPU_REQUEST]" + memory: "[MEMORY_REQUEST]" + nvidia.com/gpu: "[GPU_COUNT]" + tolerations: + - key: "nvidia.com/gpu" + operator: "Exists" + effect: "NoSchedule" + - key: "ai-app" + operator: "Equal" + value: "true" + effect: "NoSchedule" + # Add additional tolerations for cluster-specific taints as needed +``` + +**Key notes:** +- `storageUri` uses `hf://` for HuggingFace models (no auth required for public models) +- Always include GPU tolerations — production GPU nodes are almost always tainted +- Add cluster-specific tolerations discovered via `resources_list` Nodes +- The `deploymentMode: RawDeployment` annotation is typical; use `Serverless` if Knative is available and TrustyAI payload logging is needed + +## InferenceService (NIM) + +**Replaces**: `deploy_model` (from rhoai) for NIM deployments — the RHOAI tool does not support NIM-specific env vars, image pull secrets, or the `nim://` URI scheme + +**MCP Tool**: `resources_create_or_update` (from openshift) + +```yaml +apiVersion: serving.kserve.io/v1beta1 +kind: InferenceService +metadata: + name: [MODEL_NAME] + namespace: [NAMESPACE] + labels: + opendatahub.io/dashboard: "true" + annotations: + serving.kserve.io/deploymentMode: RawDeployment + openshift.io/display-name: [DISPLAY_NAME] +spec: + predictor: + model: + modelFormat: + name: vLLM + runtime: [NIM_SERVING_RUNTIME_NAME] + resources: + limits: + cpu: "[CPU_LIMIT]" + memory: "[MEMORY_LIMIT]" + nvidia.com/gpu: "[GPU_COUNT]" + requests: + cpu: "[CPU_REQUEST]" + memory: "[MEMORY_REQUEST]" + nvidia.com/gpu: "[GPU_COUNT]" + containers: + - name: kserve-container + image: "nvcr.io/nim/meta/[NIM_MODEL_IMAGE]:[TAG]" + env: + - name: NGC_API_KEY + valueFrom: + secretKeyRef: + name: ngc-api-key + key: NGC_API_KEY + - name: NIM_MAX_MODEL_LEN + value: "[MAX_MODEL_LEN]" + resources: + limits: + nvidia.com/gpu: "[GPU_COUNT]" + imagePullSecrets: + - name: ngc-image-pull-secret + tolerations: + - key: "nvidia.com/gpu" + operator: "Exists" + effect: "NoSchedule" +``` + +**Key notes:** +- NIM requires NGC API key secret and image pull secret (created by `/nim-setup`) +- Use a specific image tag (e.g., `1.8.3`) instead of `latest` to avoid CUDA driver incompatibility +- `NIM_MAX_MODEL_LEN` prevents KV cache OOM (default can be too high; use `16384` for T4 GPUs) + +## Toleration Post-Deploy Patch + +**When**: After `deploy_model` (from rhoai) succeeds but the pod is stuck Pending due to GPU node taints + +**Step 1**: Discover taints on GPU nodes: + +**MCP Tool**: `resources_list` (from openshift) +- `apiVersion`: `v1`, `kind`: `Node`, `labelSelector`: `nvidia.com/gpu.present=true` + +Extract `.spec.taints[]` from each node to identify required tolerations. + +**Step 2**: Patch InferenceService with tolerations: + +**MCP Tool**: `resources_create_or_update` (from openshift) + +Merge the tolerations into `spec.predictor.tolerations` of the existing InferenceService. + +## Notebook CR (Workbench) + +**Replaces**: `create_workbench` (from rhoai) + +**MCP Tool**: `resources_create_or_update` (from openshift) + +```yaml +apiVersion: kubeflow.org/v1 +kind: Notebook +metadata: + name: [WORKBENCH_NAME] + namespace: [NAMESPACE] + labels: + app: [WORKBENCH_NAME] + opendatahub.io/dashboard: "true" + opendatahub.io/odh-managed: "true" + annotations: + notebooks.opendatahub.io/inject-oauth: "true" + opendatahub.io/image-display-name: [IMAGE_DISPLAY_NAME] + openshift.io/display-name: [DISPLAY_NAME] +spec: + template: + spec: + containers: + - name: [WORKBENCH_NAME] + image: [FULL_IMAGE_REFERENCE] + resources: + limits: + cpu: "[CPU_LIMIT]" + memory: "[MEMORY_LIMIT]" + requests: + cpu: "[CPU_REQUEST]" + memory: "[MEMORY_REQUEST]" + volumeMounts: + - mountPath: /opt/app-root/src + name: [WORKBENCH_NAME] + env: + - name: NOTEBOOK_ARGS + value: "--ServerApp.port=8888 --ServerApp.token='' --ServerApp.password='' --ServerApp.base_url=/notebook/[NAMESPACE]/[WORKBENCH_NAME]" + volumes: + - name: [WORKBENCH_NAME] + persistentVolumeClaim: + claimName: [PVC_NAME] +``` + +**Key notes:** +- `[FULL_IMAGE_REFERENCE]` must come from ImageStream lookup (see below), NOT from `list_notebook_images` +- The `inject-oauth: "true"` annotation enables OpenShift OAuth proxy + +## Notebook Image Discovery (ImageStream Lookup) + +**Replaces**: `list_notebook_images` (from rhoai) — the RHOAI tool returns hardcoded incorrect image names + +**Step 1**: List notebook ImageStreams: + +**MCP Tool**: `resources_list` (from openshift) +- `apiVersion`: `image.openshift.io/v1`, `kind`: `ImageStream`, `namespace`: `redhat-ods-applications` +- `labelSelector`: `opendatahub.io/notebook-image=true` + +**Step 2**: Get details for each ImageStream: + +**MCP Tool**: `resources_get` (from openshift) + +Extract from each ImageStream: +- `.metadata.name` — the actual image name (e.g., `pytorch`, NOT `jupyter-pytorch-notebook`) +- `.spec.tags[].name` — available tags (e.g., `2024.1`, `2024.2`) +- `.spec.tags[].annotations["opendatahub.io/notebook-image-name"]` — display name +- `.spec.tags[].from.name` — the full image reference to use in Notebook CR + +**Step 3**: Present to user in a table: + +| Image | Tag | Display Name | +|-------|-----|-------------| +| pytorch | 2024.1 | PyTorch | +| tensorflow | 2024.1 | TensorFlow | +| minimal-notebook | 2024.1 | Standard Data Science | + +## Workbench Start/Stop (Annotation Patch) + +**Replaces**: `start_workbench` / `stop_workbench` / `manage_resource(action=stop)` (from rhoai) + +**Stop workbench**: + +**MCP Tool**: `resources_create_or_update` (from openshift) + +Patch the Notebook CR to add the stop annotation: +```yaml +metadata: + annotations: + kubeflow-resource-stopped: "true" +``` + +**Start workbench**: + +**MCP Tool**: `resources_create_or_update` (from openshift) + +Patch the Notebook CR to remove the stop annotation by setting it to empty string or removing it: +```yaml +metadata: + annotations: + kubeflow-resource-stopped: null +``` + +If `resources_create_or_update` cannot remove annotations, use the full Notebook CR manifest without the `kubeflow-resource-stopped` annotation. + +## PVC for Workbench Storage + +**Replaces**: `create_storage` (from rhoai) + +**MCP Tool**: `resources_create_or_update` (from openshift) + +```yaml +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: [PVC_NAME] + namespace: [NAMESPACE] + labels: + opendatahub.io/dashboard: "true" +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: [SIZE] +``` + +**Listing PVCs** (replaces `list_storage`): + +**MCP Tool**: `resources_list` (from openshift) +- `apiVersion`: `v1`, `kind`: `PersistentVolumeClaim`, `namespace`: `[NAMESPACE]` + +## DataSciencePipelinesApplication (DSPA) + +**Replaces**: `create_pipeline_server` (from rhoai) — the RHOAI tool constructs an invalid manifest that returns "Unprocessable Entity" + +**MCP Tool**: `resources_create_or_update` (from openshift) + +```yaml +apiVersion: datasciencepipelinesapplications.opendatahub.io/v1alpha1 +kind: DataSciencePipelinesApplication +metadata: + name: dspa + namespace: [NAMESPACE] +spec: + dspVersion: v2 + objectStorage: + disableHealthCheck: false + enableExternalRoute: false + externalStorage: + bucket: [BUCKET_NAME] + host: [S3_HOST_WITHOUT_PROTOCOL] + port: "" + s3CredentialsSecret: + accessKey: AWS_ACCESS_KEY_ID + secretKey: AWS_SECRET_ACCESS_KEY + secretName: [DATA_CONNECTION_SECRET_NAME] + scheme: [http_or_https] + region: [REGION] +``` + +**Key notes:** +- `host` must NOT include protocol prefix — use `minio.namespace.svc:9000` not `http://minio.namespace.svc:9000` +- `scheme` is separate: `http` or `https` +- `secretName` references the S3 data connection secret created earlier +- `s3CredentialsSecret` key names (`accessKey`, `secretKey`) are the keys inside the secret, not the actual credentials + +**Checking DSPA status** (replaces `get_pipeline_server`): + +**MCP Tool**: `resources_get` (from openshift) +- `apiVersion`: `datasciencepipelinesapplications.opendatahub.io/v1alpha1`, `kind`: `DataSciencePipelinesApplication`, `name`: `dspa`, `namespace`: `[NAMESPACE]` +- Check `.status.conditions` for `Ready=True` + +## ServingRuntime + +**Replaces**: `create_serving_runtime` (from rhoai) + +**Listing runtimes** (replaces `list_serving_runtimes`): + +**MCP Tool**: `resources_list` (from openshift) +- Namespace-scoped: `apiVersion`: `serving.kserve.io/v1alpha1`, `kind`: `ServingRuntime`, `namespace`: `[NAMESPACE]` +- Platform templates: `apiVersion`: `serving.kserve.io/v1alpha1`, `kind`: `ClusterServingRuntime` + +Filter by label `opendatahub.io/dashboard: "true"` and check `spec.supportedModelFormats` for compatibility. + +## InferenceService Status Lookup + +**Replaces**: `get_inference_service` / `get_model_endpoint` (from rhoai) + +**MCP Tool**: `resources_get` (from openshift) +- `apiVersion`: `serving.kserve.io/v1beta1`, `kind`: `InferenceService`, `name`: `[MODEL_NAME]`, `namespace`: `[NAMESPACE]` + +Extract: +- `.status.conditions` — check for `Ready=True` +- `.status.url` — the inference endpoint URL +- `.status.components.predictor.url` — the predictor URL + +**Listing InferenceServices** (replaces `list_inference_services`): + +**MCP Tool**: `resources_list` (from openshift) +- `apiVersion`: `serving.kserve.io/v1beta1`, `kind`: `InferenceService`, `namespace`: `[NAMESPACE]` diff --git a/rh-ai-engineer/skills/references/skill-conventions.md b/rh-ai-engineer/skills/references/skill-conventions.md index 142a73df..92e90d9b 100644 --- a/rh-ai-engineer/skills/references/skill-conventions.md +++ b/rh-ai-engineer/skills/references/skill-conventions.md @@ -1,25 +1,75 @@ --- title: Skill Conventions category: references -tags: [conventions, prerequisites, human-in-the-loop, security] -semantic_keywords: [prerequisite verification, human confirmation, credential security, skill shared patterns] -use_cases: [nim-setup, model-deploy, serving-runtime-config, debug-inference, ai-observability, model-monitor, guardrails-config] -last_updated: 2026-03-01 +tags: [conventions, prerequisites, human-in-the-loop, security, resilience, fallback] +semantic_keywords: [prerequisite verification, human confirmation, credential security, skill shared patterns, MCP resilience, OpenShift fallback, RBAC degradation] +use_cases: [nim-setup, model-deploy, serving-runtime-config, debug-inference, ai-observability, model-monitor, guardrails-config, ds-project-setup, workbench-manage, pipeline-manage, model-registry] +last_updated: 2026-03-23 --- # rh-ai-engineer Skill Conventions Shared conventions for all skills in the rh-ai-engineer agentic collection. +## MCP Server Requirement Tiers + +All rh-ai-engineer skills classify MCP servers into three tiers: + +**Required** — The skill cannot function without this server. Fail immediately if unavailable. +- `openshift` — The only universally required server. Provides reliable Kubernetes resource CRUD via `resources_get`, `resources_list`, `resources_create_or_update`, `resources_delete`, `pods_list`, `pods_log`, `events_list`. + +**Preferred** — Try this server's tools first for convenience and higher-level abstraction. On failure (auth error, API error, connection error), automatically fall back to the Required server's equivalent operation. +- `rhoai` — Provides RHOAI-specific convenience tools (`deploy_model`, `create_workbench`, etc.). All operations have OpenShift equivalents documented in [openshift-fallback-templates.md](openshift-fallback-templates.md). + +**Optional** — Used when available; features gracefully skipped when unavailable. +- `ai-observability` — GPU metrics, vLLM analysis, distributed tracing. Non-blocking. + +Skills MUST use this structure in their Prerequisites section: +```markdown +**Required MCP Server**: `openshift` (hard requirement, no fallback) +**Preferred MCP Server**: `rhoai` (used when available, OpenShift fallback on failure) +**Optional MCP Server**: `ai-observability` (skipped when unavailable) +``` + +## MCP Server Resilience Protocol + +When an RHOAI MCP tool fails (Unauthorized, Unsupported Media Type, Unprocessable Entity, connection error, or any non-success response): + +1. **Do NOT prompt the user** about the failure — immediately attempt the OpenShift fallback +2. **Use the equivalent operation** from [openshift-fallback-templates.md](openshift-fallback-templates.md) +3. **Report the fallback transparently** in the output: "Note: Used OpenShift direct API (RHOAI tool returned [brief error description])." +4. **Never fail a skill workflow** solely because an RHOAI tool is unavailable — always have an OpenShift alternative +5. **If the OpenShift fallback also fails**, then report the error and offer user options + +**RHOAI tools that are permanently replaced** (do NOT attempt these — always use OpenShift): +- `list_notebook_images` — returns incorrect hardcoded image names. Use ImageStream lookup via OpenShift instead (see [openshift-fallback-templates.md](openshift-fallback-templates.md#notebook-image-discovery-imagestream-lookup)). +- `list_catalog_sources` — wrong API type detection. Use `resources_list` for catalog CRs instead. +- `create_pipeline_server` — constructs invalid DSPA manifest. Use `resources_create_or_update` with correct DSPA YAML template instead (see [openshift-fallback-templates.md](openshift-fallback-templates.md#datasciencepipelinesapplication-dspa)). + +## RBAC Graceful Degradation + +When any MCP tool returns a 403 Forbidden or "insufficient permissions" error: + +1. **Try namespace-scoped alternative** — if a cluster-scoped operation fails (e.g., list Nodes), try the namespace-scoped equivalent (e.g., list pods with GPU resource requests) +2. **Report actionable information** — identify the specific API group, resource, and verb that was denied +3. **Never display raw HTTP errors** — translate to user-friendly messages: + ``` + Permission denied: Cannot [verb] [resource] in [namespace/cluster]. + Required RBAC: [specific role or permission needed] + Contact your cluster administrator to grant access. + ``` +4. **Suggest minimum required roles** — e.g., "edit role in namespace X" or "cluster-monitoring-view for Prometheus access" + ## Prerequisite Verification Protocol Before executing any skill, verify MCP server availability: -1. **Check MCP Server Configuration** - Verify required servers exist in `.mcp.json` -2. **Check Environment Variables** - Verify required env vars are set (check presence only, NEVER expose values) -3. **Check Optional MCP Servers** - Note availability; skip optional features if unavailable (non-blocking) +1. **Check Required MCP Servers** - Verify `openshift` server is configured and responding. Fail if unavailable. +2. **Check Preferred MCP Servers** - Note if `rhoai` is available. If unavailable, inform user that OpenShift direct API will be used for all operations (non-blocking). +3. **Check Optional MCP Servers** - Note availability of `ai-observability`; skip optional features if unavailable (non-blocking). +4. **Check Environment Variables** - Verify required env vars are set (check presence only, NEVER expose values) -**When prerequisites fail:** +**When Required prerequisites fail:** 1. Stop execution immediately 2. Report the specific missing prerequisite: @@ -36,9 +86,7 @@ Before executing any skill, verify MCP server availability: 3. Offer options: "setup" (help configure now) / "skip" (skip this skill) / "abort" (stop workflow) 4. WAIT for user decision -- never proceed automatically -**Common prerequisite: OpenShift MCP Server** - -Most rh-ai-engineer skills use the `openshift` MCP server by default (some skills may treat it as optional). Always defer to each skill's **Dependencies/Prerequisites** section for whether `openshift` is required or optional: +**OpenShift MCP Server** (universally required): - Source: https://github.com/openshift/openshift-mcp-server - Required env var: `KUBECONFIG` - Setup: Add to `.mcp.json`, set `KUBECONFIG`, restart Claude Code diff --git a/rh-ai-engineer/skills/serving-runtime-config/SKILL.md b/rh-ai-engineer/skills/serving-runtime-config/SKILL.md index 581278f3..2391b3a7 100644 --- a/rh-ai-engineer/skills/serving-runtime-config/SKILL.md +++ b/rh-ai-engineer/skills/serving-runtime-config/SKILL.md @@ -24,19 +24,20 @@ Configure custom ServingRuntime custom resources on Red Hat OpenShift AI. Use wh ## Prerequisites -**Required MCP Server**: `rhoai` ([RHOAI MCP Server](https://github.com/opendatahub-io/rhoai-mcp)) +**Required MCP Server**: `openshift` ([OpenShift MCP Server](https://github.com/openshift/openshift-mcp-server)) + +**Required MCP Tools** (from openshift): +- `resources_get` - Inspect existing ServingRuntime CRs in detail +- `resources_list` - List ServingRuntime and ClusterServingRuntime CRs (OpenShift fallback) +- `resources_create_or_update` - Create fully custom ServingRuntime CR (when not using templates, or as fallback) + +**Preferred MCP Server**: `rhoai` ([RHOAI MCP Server](https://github.com/opendatahub-io/rhoai-mcp)) — used when available, automatic OpenShift fallback on failure -**Required MCP Tools** (from rhoai): +**Preferred MCP Tools** (from rhoai): - `list_serving_runtimes` - List available runtimes and platform templates with supported model formats - `create_serving_runtime` - Instantiate a serving runtime from a platform template (no YAML needed) - `list_data_science_projects` - Validate namespace is an RHOAI project -**Required MCP Server**: `openshift` ([OpenShift MCP Server](https://github.com/openshift/openshift-mcp-server)) - -**Required MCP Tools** (from openshift): -- `resources_get` (from openshift) - Inspect existing ServingRuntime CRs in detail -- `resources_create_or_update` (from openshift) - Create fully custom ServingRuntime CR (when not using templates) - **Optional MCP Server**: `ai-observability` ([AI Observability MCP](https://github.com/rh-ai-quickstart/ai-observability-summarizer)) **Optional MCP Tools** (from ai-observability): @@ -44,6 +45,8 @@ Configure custom ServingRuntime custom resources on Red Hat OpenShift AI. Use wh **Common prerequisites** (KUBECONFIG, OpenShift+RHOAI cluster, KServe, verification protocol): See [skill-conventions.md](../references/skill-conventions.md). +**Fallback templates**: See [openshift-fallback-templates.md](../references/openshift-fallback-templates.md) for OpenShift YAML templates used when RHOAI tools are unavailable. + ## When to Use This Skill **Use this skill when you need to:** @@ -70,6 +73,8 @@ Configure custom ServingRuntime custom resources on Red Hat OpenShift AI. Use wh Verify the user-specified namespace is an RHOAI Data Science Project. +**If rhoai unavailable or returns error**: Use `resources_list` (from openshift) with `apiVersion: v1`, `kind: Namespace`, `labelSelector: opendatahub.io/dashboard=true`. + **Error Handling**: - If namespace not found in project list -> Report: "Namespace `[namespace]` is not an RHOAI Data Science Project. Use `/ds-project-setup` to create one, or specify a different namespace." **WAIT for user decision.** @@ -89,6 +94,8 @@ Verify the user-specified namespace is an RHOAI Data Science Project. - `namespace`: validated namespace from Step 1 - REQUIRED - `include_templates`: `true` - REQUIRED (shows both existing runtimes and platform templates) +**If rhoai unavailable or returns error**: Use `resources_list` (from openshift) with `apiVersion: serving.kserve.io/v1alpha1`, `kind: ServingRuntime`, `namespace: [namespace]` for namespace runtimes, and `kind: ClusterServingRuntime` for platform templates. Filter by label `opendatahub.io/dashboard=true` and check `spec.supportedModelFormats` for compatibility. + **Present findings** in a table: | Runtime Name | Model Format | Source | Requires Instantiation | @@ -204,6 +211,8 @@ Display the ServingRuntime YAML to the user, **redacting any sensitive values**. The response includes the created runtime name, display name, and supported model formats. +**If rhoai unavailable or returns error**: Use `resources_get` (from openshift) to fetch the ClusterServingRuntime template, copy its spec to a namespace-scoped ServingRuntime, and create via `resources_create_or_update` (from openshift). See [openshift-fallback-templates.md](../references/openshift-fallback-templates.md#servingruntime) for the pattern. + **If creating a fully custom runtime** (custom container image, non-template configuration): **MCP Tool**: `resources_create_or_update` (from openshift) @@ -228,6 +237,8 @@ The response includes the created runtime name, display name, and supported mode Verify the runtime appears in the namespace runtime list. +**If rhoai unavailable or returns error**: Use `resources_list` (from openshift) with `apiVersion: serving.kserve.io/v1alpha1`, `kind: ServingRuntime`, `namespace: [namespace]` for namespace runtimes, and `kind: ClusterServingRuntime` for platform templates. Filter by label `opendatahub.io/dashboard=true` and check `spec.supportedModelFormats` for compatibility. + For detailed inspection: **MCP Tool**: `resources_get` (from openshift) diff --git a/rh-ai-engineer/skills/workbench-manage/SKILL.md b/rh-ai-engineer/skills/workbench-manage/SKILL.md index d585438f..bc2294f3 100644 --- a/rh-ai-engineer/skills/workbench-manage/SKILL.md +++ b/rh-ai-engineer/skills/workbench-manage/SKILL.md @@ -28,30 +28,37 @@ Create and manage Jupyter notebook workbenches on Red Hat OpenShift AI. Handles ## Prerequisites -**Required MCP Server**: `rhoai` ([RHOAI MCP Server](https://github.com/opendatahub-io/rhoai-mcp)) +**Required MCP Server**: `openshift` ([OpenShift MCP Server](https://github.com/openshift/openshift-mcp-server)) + +**Required MCP Tools** (from openshift): +- `resources_get` - Inspect Notebook CR details, ImageStream details, check node GPU availability +- `resources_list` - List ImageStreams for notebook image discovery, list PVCs, list Notebooks +- `resources_create_or_update` - Create/update Notebook CR, PVC, patch annotations for start/stop (OpenShift fallback) +- `resources_delete` - Delete Notebook CR, PVC (OpenShift fallback for delete operations) +- `events_list` - Check pod events when workbench is stuck +- `pods_list` - Check workbench pod status + +**Preferred MCP Server**: `rhoai` ([RHOAI MCP Server](https://github.com/opendatahub-io/rhoai-mcp)) — used when available, automatic OpenShift fallback on failure -**Required MCP Tools** (from rhoai): +**Preferred MCP Tools** (from rhoai): - `list_data_science_projects` - Validate namespace is an RHOAI Data Science Project -- `list_notebook_images` - List available notebook container images (PyTorch, TensorFlow, Standard DS, etc.) - `list_workbenches` - List existing workbenches in a project - `get_workbench` - Get workbench details (status, image, resources, storage) - `create_workbench` - Create a new Notebook CR with image, resources, and storage -- `start_workbench` - Start a stopped workbench -- `stop_workbench` - Stop a running workbench -- `delete_workbench` - Delete a workbench +- `start_workbench` - Start a stopped workbench. **Known issue**: may fail with "Unsupported Media Type" — use annotation patch fallback. +- `stop_workbench` - Stop a running workbench. **Known issue**: may fail — use annotation patch fallback. +- `delete_workbench` - Delete a workbench. **Known issue**: may return "Dangerous operations are disabled" — use `resources_delete` fallback. - `get_workbench_url` - Get the OAuth-protected notebook URL - `list_storage` - List PVCs in the project - `create_storage` - Create a PVC for workbench storage - `delete_storage` - Delete a PVC - `list_data_connections` - List data connections available to attach -**Required MCP Server**: `openshift` ([OpenShift MCP Server](https://github.com/openshift/openshift-mcp-server)) +**Common prerequisites** (KUBECONFIG, OpenShift+RHOAI cluster, verification protocol): See [skill-conventions.md](../references/skill-conventions.md). -**Required MCP Tools** (from openshift): -- `resources_get` (from openshift) - Inspect Notebook CR details, check node GPU availability -- `events_list` (from openshift) - Check pod events when workbench is stuck +**Fallback templates**: See [openshift-fallback-templates.md](../references/openshift-fallback-templates.md) for OpenShift YAML templates used when RHOAI tools are unavailable. -**Common prerequisites** (KUBECONFIG, OpenShift+RHOAI cluster, verification protocol): See [skill-conventions.md](../references/skill-conventions.md). +**Important**: Do NOT use `list_notebook_images` (from rhoai) — it returns incorrect hardcoded image names that cause broken deployments. Always use the ImageStream lookup pattern described below. **Additional cluster requirements**: - Target namespace is an RHOAI Data Science Project (label: `opendatahub.io/dashboard: "true"`) @@ -95,22 +102,37 @@ Verify the user-specified namespace appears in the project list. If not, report: **Route to the appropriate sub-workflow:** - Create -> Step 2 - Start/Stop -> Step 5 -- List -> Use `list_workbenches`, display results, done +- List -> Use `list_workbenches` (fallback: `resources_list` from openshift), display results, done - Delete -> Step 6 ### Step 2: Gather Configuration (Create) -**List available notebook images:** +**Notebook Image Discovery** (replaces `list_notebook_images` which returns incorrect names): -**MCP Tool**: `list_notebook_images` (from rhoai) +**Step 1**: List notebook ImageStreams: -**Parameters**: none +**MCP Tool**: `resources_list` (from openshift) +- `apiVersion`: `image.openshift.io/v1`, `kind`: `ImageStream`, `namespace`: `redhat-ods-applications`, `labelSelector`: `opendatahub.io/notebook-image=true` + +**Step 2**: For each ImageStream, get details: + +**MCP Tool**: `resources_get` (from openshift) + +Extract from each ImageStream: +- `.metadata.name` — the actual image name (e.g., `pytorch`, NOT `jupyter-pytorch-notebook`) +- `.spec.tags[].name` — available tags (e.g., `2024.1`) +- `.spec.tags[].annotations["opendatahub.io/notebook-image-name"]` — display name +- `.spec.tags[].from.name` — the full image reference to use in the Notebook CR + +**Present to user** as a selection table showing Image Name, Tag, and Display Name. + +See [openshift-fallback-templates.md](../references/openshift-fallback-templates.md#notebook-image-discovery-imagestream-lookup) for the complete pattern. **Present available images** in a table: -| Image Name | Description | -|------------|-------------| -| [name] | [description] | +| Image Name | Tag | Display Name | +|------------|-----|--------------| +| [name] | [tag] | [display_name] | **Ask the user for workbench configuration:** - **Workbench name**: DNS-compatible name (lowercase, hyphens, max 63 chars) @@ -143,6 +165,8 @@ Verify the user-specified namespace appears in the project list. If not, report: **Parameters**: - `namespace`: target namespace - REQUIRED +**If rhoai unavailable or returns error**: Use `resources_list`/`resources_create_or_update`/`resources_delete` (from openshift) for PersistentVolumeClaim resources. See [openshift-fallback-templates.md](../references/openshift-fallback-templates.md#pvc-for-workbench-storage). + If a suitable PVC already exists, ask user if they want to reuse it or create a new one. **Create PVC for workbench storage:** @@ -155,6 +179,8 @@ If a suitable PVC already exists, ask user if they want to reuse it or create a - `size`: storage size from Step 2 (e.g., `"20Gi"`) - REQUIRED - `access_mode`: `"ReadWriteOnce"` - REQUIRED (default, single-pod access) +**If rhoai unavailable or returns error**: Use `resources_list`/`resources_create_or_update`/`resources_delete` (from openshift) for PersistentVolumeClaim resources. See [openshift-fallback-templates.md](../references/openshift-fallback-templates.md#pvc-for-workbench-storage). + **Verify creation:** **MCP Tool**: `list_storage` (from rhoai) @@ -201,7 +227,7 @@ Check until status shows the workbench is running. If status does not become rea **Error Handling**: - If workbench name already exists -> Report: "Workbench `[name]` already exists. Choose a different name or manage the existing one." -- If image not found -> Re-run `list_notebook_images` and suggest available alternatives +- If image not found -> Re-run the ImageStream lookup pattern and suggest available alternatives - If RBAC error -> Report insufficient permissions to create Notebook CRs - If GPU unavailable -> Report: "Requested GPU resources not available on cluster nodes. Reduce GPU count or wait for resources." @@ -230,6 +256,8 @@ Check until status shows the workbench is running. If status does not become rea **Parameters**: - `namespace`: target namespace - REQUIRED +**If rhoai unavailable or returns error**: Use `resources_list` (from openshift) with `apiVersion: kubeflow.org/v1`, `kind: Notebook`, `namespace: [namespace]`. + If user did not specify a workbench name, present the list and ask which one to manage. **For Start:** @@ -242,6 +270,8 @@ Confirm the workbench is currently stopped. If already running, report its URL a - `namespace`: target namespace - REQUIRED - `name`: workbench name - REQUIRED +**If rhoai unavailable or returns error (e.g., "Unsupported Media Type")**: Patch the Notebook CR annotation via `resources_create_or_update` (from openshift) to remove the `kubeflow-resource-stopped` annotation (set to null or empty). See [openshift-fallback-templates.md](../references/openshift-fallback-templates.md#workbench-startstop-annotation-patch). + **MCP Tool**: `get_workbench_url` (from rhoai) **Parameters**: @@ -260,6 +290,8 @@ Confirm the workbench is currently stopped. If already running, report its URL a - `namespace`: target namespace - REQUIRED - `name`: workbench name - REQUIRED +**If rhoai unavailable or returns error**: Patch the Notebook CR via `resources_create_or_update` (from openshift) to set annotation `kubeflow-resource-stopped: "true"`. See [openshift-fallback-templates.md](../references/openshift-fallback-templates.md#workbench-startstop-annotation-patch). + **Verify state change:** **MCP Tool**: `get_workbench` (from rhoai) @@ -268,6 +300,8 @@ Confirm the workbench is currently stopped. If already running, report its URL a - `namespace`: target namespace - REQUIRED - `name`: workbench name - REQUIRED +**If rhoai unavailable or returns error**: Use `resources_get` (from openshift) with `apiVersion: kubeflow.org/v1`, `kind: Notebook`, `name: [name]`, `namespace: [namespace]`. + **Output to user**: "Workbench `[name]` stopped. Persistent storage is preserved. Use `/workbench-manage` to start it again." **Error Handling**: @@ -284,6 +318,8 @@ Confirm the workbench is currently stopped. If already running, report its URL a - `namespace`: target namespace - REQUIRED - `name`: workbench name - REQUIRED +**If rhoai unavailable or returns error**: Use `resources_get` (from openshift) with `apiVersion: kubeflow.org/v1`, `kind: Notebook`, `name: [name]`, `namespace: [namespace]`. + **Display workbench details and data loss warning:** | Detail | Value | @@ -305,6 +341,8 @@ Confirm the workbench is currently stopped. If already running, report its URL a - `namespace`: target namespace - REQUIRED - `name`: workbench name - REQUIRED +**If rhoai unavailable or returns error (e.g., "Dangerous operations are disabled")**: Use `resources_delete` (from openshift) with `apiVersion: kubeflow.org/v1`, `kind: Notebook`, `name: [workbench-name]`, `namespace: [namespace]`. WAIT for user confirmation before deleting — warn about data loss. + **Associated storage cleanup** (separate confirmation): **Ask**: "The PVC `[pvc_name]` ([size]) associated with this workbench still exists. Delete it too? WARNING: All data in this volume will be permanently lost. (yes/no)" @@ -319,6 +357,8 @@ If user confirms PVC deletion: - `namespace`: target namespace - REQUIRED - `name`: PVC name - REQUIRED +**If rhoai unavailable or returns error**: Use `resources_list`/`resources_create_or_update`/`resources_delete` (from openshift) for PersistentVolumeClaim resources. See [openshift-fallback-templates.md](../references/openshift-fallback-templates.md#pvc-for-workbench-storage). + If user declines, report: "PVC `[pvc_name]` preserved. It can be reattached to a new workbench." **Output to user**: "Workbench `[name]` deleted. [PVC deleted / PVC preserved]." @@ -334,10 +374,20 @@ For common issues (GPU scheduling, OOMKilled, image pull errors, RBAC), see [com **Cause**: The selected image name does not match any available notebook image, or the image registry is unreachable. **Solution:** -1. Run `list_notebook_images` to see current available images +1. Run the ImageStream lookup pattern to see current available images 2. Verify the exact image name (case-sensitive) 3. If no images are listed, the RHOAI operator may not have imported notebook images -- contact cluster administrator +### Issue: Workbench Created with Wrong Image (ImagePullBackOff) + +**Error**: Workbench pod stuck in `ImagePullBackOff` after creation + +**Cause**: The `list_notebook_images` tool returned incorrect image names (e.g., `jupyter-pytorch-notebook` instead of the actual ImageStream name `pytorch`). + +**Solution**: This tool has been replaced. Use the ImageStream lookup pattern via OpenShift MCP to discover correct image names. Patch the stuck Notebook CR with the correct image reference from the ImageStream, then delete the stuck pod to force rescheduling. + +See [common-issues.md](../references/common-issues.md#notebook-image-names-mismatch) for details. + ### Issue 2: PVC Binding Failure **Error**: PVC remains in `Pending` state, workbench cannot start From 518f3cacf75ea049993a9bad07963f2bd6a15627 Mon Sep 17 00:00:00 2001 From: ikrispin Date: Mon, 23 Mar 2026 14:36:21 +0200 Subject: [PATCH 2/7] refactor: introduce CLAUDE.md file for instruction routing and global plugin rules --- rh-ai-engineer/CLAUDE.md | 57 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 rh-ai-engineer/CLAUDE.md diff --git a/rh-ai-engineer/CLAUDE.md b/rh-ai-engineer/CLAUDE.md new file mode 100644 index 00000000..db6cdbcc --- /dev/null +++ b/rh-ai-engineer/CLAUDE.md @@ -0,0 +1,57 @@ +# rh-ai-engineer Plugin + +You are an AI/ML engineer assistant for Red Hat OpenShift AI (RHOAI). You help users deploy models, manage workbenches, configure pipelines, set up monitoring, and operate AI infrastructure on OpenShift clusters. + +## Skill-First Rule + +ALWAYS use the appropriate skill for RHOAI tasks. Do NOT call MCP tools (rhoai, openshift, ai-observability) directly — skills handle error recovery, OpenShift fallbacks, credential safety, and user confirmations automatically. + +To invoke a skill, use the Skill tool with the skill name (e.g., `/model-deploy`). + +## Intent Routing + +Match the user's request to the correct skill: + +| When the user asks about... | Use skill | +|----------------------------|-----------| +| Creating a project, namespace, data connection, S3 storage, pipeline server setup, enable model serving | `/ds-project-setup` | +| Workbench, notebook, Jupyter, start/stop workbench, notebook images | `/workbench-manage` | +| Deploy model, serve model, inference endpoint, vLLM, KServe, InferenceService, Granite, Llama | `/model-deploy` | +| Model registry, register model, model versions, promote model, model catalog | `/model-registry` | +| Pipeline, pipeline run, schedule pipeline, Kubeflow, DSPA, pipeline logs | `/pipeline-manage` | +| NIM, NGC credentials, NIM setup, NVIDIA NIM platform | `/nim-setup` | +| Serving runtime, custom runtime, ServingRuntime, runtime template | `/serving-runtime-config` | +| Debug deployment, model not starting, stuck deployment, inference errors, slow model | `/debug-inference` | +| GPU metrics, model performance, latency, throughput, cluster health, Prometheus, traces | `/ai-observability` | +| Bias detection, drift monitoring, TrustyAI, fairness metrics, SPD, DIR | `/model-monitor` | +| Guardrails, content safety, PII detection, prompt injection, toxicity filter | `/guardrails-config` | + +If the request doesn't clearly match one skill, ask the user to clarify. + +## Skill Chaining + +Some workflows require multiple skills in sequence: + +- **NIM model deployment**: Run `/nim-setup` first (one-time), then `/model-deploy` +- **New project bootstrap**: `/ds-project-setup` → `/workbench-manage` or `/model-deploy` +- **Post-deployment monitoring**: `/model-deploy` → `/ai-observability` → `/model-monitor` +- **Content safety setup**: `/model-deploy` → `/guardrails-config` +- **Debugging a failed deployment**: `/debug-inference`, then `/model-deploy` to fix and redeploy + +After completing a skill, suggest relevant next-step skills to the user. + +## MCP Servers + +Three MCP servers are available. Skills manage these automatically — do not call their tools directly. + +- **openshift** (Required) — Kubernetes resource CRUD, pod logs, events. The reliable foundation. +- **rhoai** (Preferred) — RHOAI-specific convenience tools. May return auth errors; skills fall back to openshift automatically. +- **ai-observability** (Optional) — GPU metrics, vLLM analysis, distributed tracing. Skipped if unavailable. + +## Global Rules + +1. **Never expose credentials** — do not display API keys, passwords, tokens, or secret values in output. Only report whether they exist. +2. **Confirm before creating resources** — always show the resource manifest (with credentials redacted) and wait for explicit user approval before creating, modifying, or deleting cluster resources. +3. **Never auto-delete** — destructive operations (delete workbench, delete model, delete pipeline) always require user confirmation with a data-loss warning. +4. **Report fallbacks transparently** — if a preferred tool fails and an OpenShift fallback is used, briefly note it (e.g., "Used OpenShift direct API for this operation"). +5. **Suggest next steps** — after completing a skill, suggest related skills the user might want to run next. From a2ae101d26b85682f4ac3ca87ecd441bcf2aea5e Mon Sep 17 00:00:00 2001 From: ikrispin Date: Mon, 23 Mar 2026 16:02:55 +0200 Subject: [PATCH 3/7] refactor: introduce CLAUDE.md file for instruction routing and global plugin rules --- rh-developer/CLAUDE.md | 64 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 rh-developer/CLAUDE.md diff --git a/rh-developer/CLAUDE.md b/rh-developer/CLAUDE.md new file mode 100644 index 00000000..d3724294 --- /dev/null +++ b/rh-developer/CLAUDE.md @@ -0,0 +1,64 @@ +# rh-developer Plugin + +You are an application developer assistant for Red Hat platforms. You help users build, containerize, deploy, and troubleshoot applications on OpenShift clusters and standalone RHEL/Fedora/CentOS systems. + +## Skill-First Rule + +ALWAYS use the appropriate skill for developer tasks. Do NOT call MCP tools (openshift, podman, github, lightspeed-mcp) directly — skills handle error recovery, human-in-the-loop confirmations, and fallbacks automatically. + +To invoke a skill, use the Skill tool with the skill name (e.g., `/deploy`). + +## Intent Routing + +Match the user's request to the correct skill: + +| When the user asks about... | Use skill | +|---|---| +| Detect language, framework, analyze project, scan repo, identify runtime | `/detect-project` | +| Recommend builder image, S2I image, base image, image selection | `/recommend-image` | +| S2I build, source-to-image, BuildConfig, build container image | `/s2i-build` | +| Deploy to OpenShift, create Deployment, Service, Route, expose app | `/deploy` | +| Helm chart, Helm deploy, Helm install, Helm values, chart template | `/helm-deploy` | +| Deploy to RHEL, Fedora, CentOS, SSH deploy, systemd service, Podman on RHEL | `/rhel-deploy` | +| End-to-end deployment, containerize and deploy, full workflow, deploy from source | `/containerize-deploy` | +| Build failure, BuildConfig error, S2I error, build logs, failed build | `/debug-build` | +| Pod failure, CrashLoopBackOff, ImagePullBackOff, OOMKilled, Pending pod | `/debug-pod` | +| Container issue, Podman/Docker failure, local container debug, container crash | `/debug-container` | +| Network issue, DNS, Service connectivity, Route, NetworkPolicy, ingress | `/debug-network` | +| Pipeline failure, Tekton, PipelineRun, TaskRun error, pipeline logs | `/debug-pipeline` | +| RHEL issue, systemd, SELinux, firewall, journal logs, system service | `/debug-rhel` | +| Check tools, verify cluster access, validate environment, prerequisites | `/validate-environment` | + +If the request doesn't clearly match one skill, ask the user to clarify. + +## Skill Chaining + +Some workflows require multiple skills in sequence: + +- **Full app deployment (S2I)**: `/detect-project` -> `/recommend-image` (optional) -> `/s2i-build` -> `/deploy` +- **Helm deployment**: `/detect-project` -> `/helm-deploy` +- **RHEL deployment**: `/detect-project` -> `/rhel-deploy` +- **Unified workflow**: `/containerize-deploy` (orchestrates all above based on user selection) +- **Pre-flight check**: Run `/validate-environment` before any deployment skill +- **Build failure recovery**: `/debug-build` -> fix -> `/s2i-build` retry +- **Pod failure recovery**: `/debug-pod` or `/debug-network` -> fix -> `/deploy` retry +- **RHEL failure recovery**: `/debug-rhel` or `/debug-container` -> fix -> `/rhel-deploy` retry + +After completing a skill, suggest relevant next-step skills to the user. + +## MCP Servers + +Four MCP servers are available. Skills manage these automatically — do not call their tools directly. + +- **openshift** (Required) — Kubernetes resource CRUD, pod logs, events, Helm operations. The reliable foundation. +- **podman** (Required) — Local container builds and image management via Podman. +- **github** (Optional) — Remote repository browsing and code analysis. Used by `/detect-project` for GitHub URLs. +- **lightspeed-mcp** (Optional) — CVE vulnerability data, advisor rules, RHEL lifecycle checks. Used by `/rhel-deploy` and `/debug-rhel`. + +## Global Rules + +1. **Never expose credentials** — do not display API keys, passwords, tokens, or secret values in output. Only report whether they exist. +2. **Confirm before creating resources** — always show the resource manifest (with credentials redacted) and wait for explicit user approval before creating, modifying, or deleting cluster or system resources. +3. **Never auto-delete** — destructive operations (delete Deployment, remove systemd service, delete BuildConfig) always require user confirmation with a data-loss warning. +4. **Report fallbacks transparently** — if a preferred tool fails and a fallback is used, briefly note it. +5. **Suggest next steps** — after completing a skill, suggest related skills the user might want to run next. From 959f73b679c3548ed02655096f08dc9310c13c39 Mon Sep 17 00:00:00 2001 From: ikrispin Date: Mon, 23 Mar 2026 16:03:50 +0200 Subject: [PATCH 4/7] refactor: introduce CLAUDE.md file for instruction routing and global plugin rules --- ocp-admin/CLAUDE.md | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 ocp-admin/CLAUDE.md diff --git a/ocp-admin/CLAUDE.md b/ocp-admin/CLAUDE.md new file mode 100644 index 00000000..d03a2f65 --- /dev/null +++ b/ocp-admin/CLAUDE.md @@ -0,0 +1,37 @@ +# ocp-admin Plugin + +You are an OpenShift cluster administrator assistant. You help users manage multi-cluster environments, generate health reports, monitor resource utilization, and operate OpenShift infrastructure. + +## Skill-First Rule + +ALWAYS use the appropriate skill for cluster administration tasks. Do NOT call MCP tools (openshift) directly — skills handle error recovery, fallbacks, credential safety, and user confirmations automatically. + +To invoke a skill, use the Skill tool with the skill name (e.g., `/cluster-report`). + +## Intent Routing + +Match the user's request to the correct skill: + +| When the user asks about... | Use skill | +|----------------------------|-----------| +| Cluster report, fleet health, multi-cluster status, compare clusters, capacity planning, cluster overview, node resources, GPU inventory | `/cluster-report` | + +If the request doesn't clearly match a skill, ask the user to clarify. + +## Skill Chaining + +After completing a skill, suggest relevant next steps to the user. + +## MCP Servers + +One MCP server is available. Skills manage it automatically — do not call its tools directly. + +- **openshift** (Required) — Kubernetes resource CRUD, pod logs, events, multi-cluster support. Runs in read-only mode inside a container. + +## Global Rules + +1. **Never expose credentials** — do not display KUBECONFIG paths, tokens, API keys, or secret values in output. Only report whether they are set. +2. **Confirm before creating resources** — always show the resource manifest (with credentials redacted) and wait for explicit user approval before creating, modifying, or deleting cluster resources. +3. **Never auto-delete** — destructive operations always require user confirmation with a data-loss warning. +4. **Report fallbacks transparently** — if a preferred tool fails and a fallback is used (e.g., `namespaces_list` instead of `projects_list`), briefly note it. +5. **Suggest next steps** — after completing a skill, suggest related actions the user might want to take next. From 2d1b39bbb92fbf7ade88fa3c42b88aed279c6a66 Mon Sep 17 00:00:00 2001 From: ikrispin Date: Mon, 23 Mar 2026 16:24:42 +0200 Subject: [PATCH 5/7] refactor: enforce CLAUDE.md as a validated requirement for all packs with skills --- CLAUDE.md | 15 +++++---- SKILL_DESIGN_PRINCIPLES.md | 19 +++++++++++ scripts/validate_structure.py | 60 +++++++++++++++++++++++++++++++++++ 3 files changed, 88 insertions(+), 6 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index cb1f6727..54689450 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -22,6 +22,7 @@ agentic-collections/ Each pack follows this structure: ``` / +├── CLAUDE.md # Claude Code instruction routing (persona, skills, rules) ├── README.md # Pack description, persona, target marketplaces ├── .claude-plugin/ # Claude Code plugin metadata │ └── plugin.json # Name, version, description, author, license @@ -150,10 +151,11 @@ last_updated: YYYY-MM-DD 1. Create pack folder: `/` 2. Add `README.md` with description, persona, marketplaces -3. Create `skills/` directory -4. Optional: Add `.claude-plugin/plugin.json` for Claude Code -5. Optional: Add `.mcp.json` for MCP server integrations -6. Update main `README.md` table with link +3. Add `CLAUDE.md` with persona, skill-first rule, intent routing table, MCP servers, and global rules (see [rh-ai-engineer/CLAUDE.md](rh-ai-engineer/CLAUDE.md) for reference) +4. Create `skills/` directory +5. Optional: Add `.claude-plugin/plugin.json` for Claude Code +6. Optional: Add `.mcp.json` for MCP server integrations +7. Update main `README.md` table with link ### Adding a Skill @@ -168,8 +170,9 @@ last_updated: YYYY-MM-DD - Workflow with precise parameters - Dependencies declaration 4. Include concrete examples and complete error handling -5. Test with `Skill` tool invocation -6. Validate with `./scripts/run-skill-linter.sh skills//` +5. Update the pack's `CLAUDE.md` intent routing table to include the new skill +6. Test with `Skill` tool invocation +7. Validate with `./scripts/run-skill-linter.sh skills//` **Collection-Specific Standards:** - **rh-virt**: Follow `rh-virt/SKILL_TEMPLATE.md` for enhanced quality standards including mandatory Common Issues and Example Usage sections diff --git a/SKILL_DESIGN_PRINCIPLES.md b/SKILL_DESIGN_PRINCIPLES.md index 9eb31af7..3d791d2e 100644 --- a/SKILL_DESIGN_PRINCIPLES.md +++ b/SKILL_DESIGN_PRINCIPLES.md @@ -361,6 +361,24 @@ One clear purpose per skill. --- +### 11. Pack-Level CLAUDE.md + +Every pack with skills MUST have a `CLAUDE.md` in its root directory. This file acts as the instruction router for Claude Code. + +**Required Sections:** +- `## Skill-First Rule` — enforce skill invocation over direct MCP tool calls +- `## Intent Routing` — table mapping user intents to skill names +- `## MCP Servers` — list available MCP servers with descriptions +- `## Global Rules` — credential safety, confirmation requirements, next-step suggestions + +**When adding a new skill**, update the pack's `CLAUDE.md` intent routing table to include it. + +**Reference:** [rh-ai-engineer/CLAUDE.md](rh-ai-engineer/CLAUDE.md) + +**Validated by:** `scripts/validate_structure.py` (automated — checks existence, required sections, and intent routing completeness) + +--- + ## Root-Level Frontmatter (2026 Standard) UI/runtime fields at root; custom fields in `metadata`. @@ -530,6 +548,7 @@ Ask: "Proceed?" Wait for confirmation. 8. **Single Responsibility** - One purpose per skill 9. **Naming Conventions** - kebab-case 10. **Content Quality** - Production-ready examples +11. **Pack-Level CLAUDE.md** - Instruction routing for every pack with skills --- diff --git a/scripts/validate_structure.py b/scripts/validate_structure.py index 8a02bd8a..bdb9c568 100644 --- a/scripts/validate_structure.py +++ b/scripts/validate_structure.py @@ -177,6 +177,63 @@ def validate_agents(pack_dir: str) -> List[str]: return errors +CLAUDE_MD_REQUIRED_SECTIONS = [ + "Skill-First Rule", + "Intent Routing", + "MCP Servers", + "Global Rules", +] + + +def validate_claude_md(pack_dir: str) -> List[str]: + """ + Validate CLAUDE.md presence and structure. + + Required for any pack that has skills. Checks for required sections + and verifies that all skills appear in the intent routing content. + + Args: + pack_dir: Pack directory name + + Returns: + List of error messages (empty if valid) + """ + errors = [] + claude_path = Path(pack_dir) / 'CLAUDE.md' + skills_dir = Path(pack_dir) / 'skills' + + has_skills = skills_dir.exists() and any(skills_dir.glob('*/SKILL.md')) + + if not claude_path.exists(): + if has_skills: + errors.append(f"{pack_dir}: Missing CLAUDE.md (required for packs with skills)") + return errors + + try: + with open(claude_path, 'r', encoding='utf-8') as f: + content = f.read() + + # Check required sections + headings = re.findall(r'^## (.+)$', content, re.MULTILINE) + for section in CLAUDE_MD_REQUIRED_SECTIONS: + if not any(section in h for h in headings): + errors.append(f"{pack_dir}: CLAUDE.md missing required section '## {section}'") + + # Check intent routing completeness + if has_skills: + skill_names = [p.parent.name for p in skills_dir.glob('*/SKILL.md')] + for skill_name in skill_names: + if skill_name not in content: + errors.append( + f"{pack_dir}: CLAUDE.md intent routing missing skill '{skill_name}'" + ) + + except Exception as e: + errors.append(f"{pack_dir}: Error reading CLAUDE.md: {e}") + + return errors + + def validate_pack(pack_dir: str) -> List[str]: """ Validate a single pack. @@ -200,6 +257,9 @@ def validate_pack(pack_dir: str) -> List[str]: # Validate .mcp.json errors.extend(validate_mcp_json(pack_dir)) + # Validate CLAUDE.md + errors.extend(validate_claude_md(pack_dir)) + # Validate skills errors.extend(validate_skills(pack_dir)) From cff4a26dd06081ffc541a5bc740dc02a1d9d5696 Mon Sep 17 00:00:00 2001 From: ikrispin Date: Tue, 24 Mar 2026 14:42:41 +0200 Subject: [PATCH 6/7] feat: validate model and color as required fields in skill frontmatter --- scripts/validate_structure.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/scripts/validate_structure.py b/scripts/validate_structure.py index bdb9c568..a636dbb4 100644 --- a/scripts/validate_structure.py +++ b/scripts/validate_structure.py @@ -86,12 +86,15 @@ def validate_mcp_json(pack_dir: str) -> List[str]: return errors -def validate_yaml_frontmatter(file_path: Path) -> Tuple[bool, str]: +def validate_yaml_frontmatter( + file_path: Path, extra_required_fields: List[str] = None +) -> Tuple[bool, str]: """ Validate YAML frontmatter in a markdown file. Args: file_path: Path to the markdown file + extra_required_fields: Additional fields to require beyond name/description Returns: Tuple of (is_valid, error_message) @@ -117,6 +120,11 @@ def validate_yaml_frontmatter(file_path: Path) -> Tuple[bool, str]: if 'description' not in data: return False, "Missing required field 'description' in frontmatter" + # Check extra required fields (e.g. model, color for skills) + for field in (extra_required_fields or []): + if field not in data: + return False, f"Missing required field '{field}' in frontmatter" + return True, "" except yaml.YAMLError as e: @@ -144,7 +152,9 @@ def validate_skills(pack_dir: str) -> List[str]: # Find all SKILL.md files for skill_file in skills_dir.glob('*/SKILL.md'): - is_valid, error_msg = validate_yaml_frontmatter(skill_file) + is_valid, error_msg = validate_yaml_frontmatter( + skill_file, extra_required_fields=['model', 'color'] + ) if not is_valid: errors.append(f"{skill_file}: {error_msg}") From 9fb52154adc3fcc548a50dd52068d9bf49a201d6 Mon Sep 17 00:00:00 2001 From: ikrispin Date: Tue, 24 Mar 2026 14:45:40 +0200 Subject: [PATCH 7/7] refactor: scope validate_structure.py to pack-level checks only, removing duplicate skill validation --- scripts/validate_structure.py | 115 ++-------------------------------- 1 file changed, 5 insertions(+), 110 deletions(-) diff --git a/scripts/validate_structure.py b/scripts/validate_structure.py index a636dbb4..1b8c6077 100644 --- a/scripts/validate_structure.py +++ b/scripts/validate_structure.py @@ -1,13 +1,15 @@ #!/usr/bin/env python3 """ -Validate agentic collection structure before documentation generation. +Validate agentic collection pack structure (plugin.json, .mcp.json, CLAUDE.md). + +Skill-level validation (frontmatter, sections, security) is handled by +validate-skills.sh and run-skill-linter.sh. """ import json import sys from pathlib import Path -from typing import List, Tuple -import yaml +from typing import List import re # List of agentic collections to validate @@ -86,107 +88,6 @@ def validate_mcp_json(pack_dir: str) -> List[str]: return errors -def validate_yaml_frontmatter( - file_path: Path, extra_required_fields: List[str] = None -) -> Tuple[bool, str]: - """ - Validate YAML frontmatter in a markdown file. - - Args: - file_path: Path to the markdown file - extra_required_fields: Additional fields to require beyond name/description - - Returns: - Tuple of (is_valid, error_message) - """ - try: - with open(file_path, 'r', encoding='utf-8') as f: - content = f.read() - - # Match YAML frontmatter - match = re.match(r'^---\s*\n(.*?)\n---\s*\n', content, re.DOTALL) - if not match: - return False, "Missing YAML frontmatter (should start with --- and end with ---)" - - frontmatter_text = match.group(1) - data = yaml.safe_load(frontmatter_text) - - if data is None: - return False, "Empty YAML frontmatter" - - # Check required fields - if 'name' not in data: - return False, "Missing required field 'name' in frontmatter" - if 'description' not in data: - return False, "Missing required field 'description' in frontmatter" - - # Check extra required fields (e.g. model, color for skills) - for field in (extra_required_fields or []): - if field not in data: - return False, f"Missing required field '{field}' in frontmatter" - - return True, "" - - except yaml.YAMLError as e: - return False, f"Invalid YAML: {e}" - except Exception as e: - return False, f"Error reading file: {e}" - - -def validate_skills(pack_dir: str) -> List[str]: - """ - Validate skills in a pack. - - Args: - pack_dir: Pack directory name - - Returns: - List of error messages (empty if valid) - """ - errors = [] - skills_dir = Path(pack_dir) / 'skills' - - if not skills_dir.exists(): - # Skills directory is optional - return errors - - # Find all SKILL.md files - for skill_file in skills_dir.glob('*/SKILL.md'): - is_valid, error_msg = validate_yaml_frontmatter( - skill_file, extra_required_fields=['model', 'color'] - ) - if not is_valid: - errors.append(f"{skill_file}: {error_msg}") - - return errors - - -def validate_agents(pack_dir: str) -> List[str]: - """ - Validate agents in a pack. - - Args: - pack_dir: Pack directory name - - Returns: - List of error messages (empty if valid) - """ - errors = [] - agents_dir = Path(pack_dir) / 'agents' - - if not agents_dir.exists(): - # Agents directory is optional - return errors - - # Find all .md files - for agent_file in agents_dir.glob('*.md'): - is_valid, error_msg = validate_yaml_frontmatter(agent_file) - if not is_valid: - errors.append(f"{agent_file}: {error_msg}") - - return errors - - CLAUDE_MD_REQUIRED_SECTIONS = [ "Skill-First Rule", "Intent Routing", @@ -270,12 +171,6 @@ def validate_pack(pack_dir: str) -> List[str]: # Validate CLAUDE.md errors.extend(validate_claude_md(pack_dir)) - # Validate skills - errors.extend(validate_skills(pack_dir)) - - # Validate agents - errors.extend(validate_agents(pack_dir)) - return errors