From a6091828c06ab3f975bc592881d7cdb795c4ff0e Mon Sep 17 00:00:00 2001 From: Jean-Pascal MEWENEMESSE Date: Wed, 8 Jul 2026 14:46:34 +0200 Subject: [PATCH 1/4] feat: restore the cluster deployment layer under clusters/sandbox The contexts improved namings files are kept reference packages at the new unversioned OCI path --- .../sandbox/contexts/10-platform-context.yaml | 151 +++ .../sandbox/contexts/20-provider-context.yaml | 57 + .../sandbox/contexts/30-service-context.yaml | 329 +++++ .../sandbox/contexts/99-examples-context.yaml | 1169 +++++++++++++++++ clusters/sandbox/flux/kubocd.yaml | 84 ++ clusters/sandbox/releases/cert-manager.yaml | 55 + clusters/sandbox/releases/cloudnative-pg.yaml | 32 + .../sandbox/releases/cnpg-postgresql.yaml | 32 + clusters/sandbox/releases/coredns-patch.yaml | 32 + clusters/sandbox/releases/dns-server.yaml | 33 + clusters/sandbox/releases/ingress-nginx.yaml | 34 + clusters/sandbox/releases/keycloak.yaml | 35 + .../releases/local-secrets-provider.yaml | 32 + clusters/sandbox/releases/okdp-server.yaml | 30 + clusters/sandbox/releases/okdp-ui.yaml | 30 + clusters/sandbox/releases/spark-operator.yaml | 17 + clusters/sandbox/releases/spark-rbac.yaml | 12 + clusters/sandbox/releases/tools.yaml | 38 + clusters/sandbox/releases/webhooks.yaml | 32 + 19 files changed, 2234 insertions(+) create mode 100644 clusters/sandbox/contexts/10-platform-context.yaml create mode 100644 clusters/sandbox/contexts/20-provider-context.yaml create mode 100644 clusters/sandbox/contexts/30-service-context.yaml create mode 100644 clusters/sandbox/contexts/99-examples-context.yaml create mode 100644 clusters/sandbox/flux/kubocd.yaml create mode 100644 clusters/sandbox/releases/cert-manager.yaml create mode 100644 clusters/sandbox/releases/cloudnative-pg.yaml create mode 100644 clusters/sandbox/releases/cnpg-postgresql.yaml create mode 100644 clusters/sandbox/releases/coredns-patch.yaml create mode 100644 clusters/sandbox/releases/dns-server.yaml create mode 100644 clusters/sandbox/releases/ingress-nginx.yaml create mode 100644 clusters/sandbox/releases/keycloak.yaml create mode 100644 clusters/sandbox/releases/local-secrets-provider.yaml create mode 100644 clusters/sandbox/releases/okdp-server.yaml create mode 100644 clusters/sandbox/releases/okdp-ui.yaml create mode 100644 clusters/sandbox/releases/spark-operator.yaml create mode 100644 clusters/sandbox/releases/spark-rbac.yaml create mode 100644 clusters/sandbox/releases/tools.yaml create mode 100644 clusters/sandbox/releases/webhooks.yaml diff --git a/clusters/sandbox/contexts/10-platform-context.yaml b/clusters/sandbox/contexts/10-platform-context.yaml new file mode 100644 index 0000000..3175eea --- /dev/null +++ b/clusters/sandbox/contexts/10-platform-context.yaml @@ -0,0 +1,151 @@ +# +# Copyright 2026 The OKDP Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +apiVersion: kubocd.kubotal.io/v1alpha1 +kind: Context +metadata: + name: platform + namespace: kubocd-system +spec: + # -- KuboCD context values for the platform layer. + context: + # -- Platform-wide defaults for ingress, certificates, storage classes, portal catalogs, exposed services, and UI metadata. + platform: + proxy: + httpProxy: ~ + httpsProxy: ~ + noProxy: ~ + ingress: + suffix: okdp.sandbox + className: nginx + + certificateIssuers: + selfSigned: + name: default-issuer + + storageClass: + data: standard + workspace: standard + + portal: + oidc: + enabled: true + provider: defaultIdp + packageRepository: quay.io/okdp/platform-packages-v0.3 + clusters: + - id: sandbox + name: "OKDP Sandbox Cluster" + env: dev + auth: + inCluster: true + catalogs: + - id: Dataviz + name: "Dataviz catalog" + description: "Dataviz packages" + packages: + - name: superset + - id: Notebook + name: "Notebooks catalog" + description: "Notebooking packages" + packages: + - name: jupyterhub + - id: Spark + name: "Spark packages" + description: "Spark packages" + packages: + - name: spark-history-server + - id: Storage + name: "Storage catalog" + description: "Storage packages" + packages: + - name: seaweedfs + - id: Interactive-Query + name: "Interactive Query" + description: "Interactive-query packages" + packages: + - name: trinodb + - id: Data-Catalog + name: "Data Catalog" + description: "Central metadata catalog packages" + packages: + - name: hive-metastore + - name: polaris + - id: Applications + name: "OKDP Examples" + description: "A collection of hands-on okdp examples" + packages: + - name: okdp-examples + - id: Workflows + name: "Workflow Orchestration" + description: "Workflow orchestration and scheduling packages" + packages: + - name: airflow + services: + - Notebook + - Spark + - Dataviz + - Interactive-Query + - Data-Catalog + - Storage + - Applications + - Workflows + kadCatalogsInfo: + Interactive-Query: + displayName: "Interactive Query" + menuIcon: "fa-solid fa-database" + Data-Catalog: + displayName: "Data Catalog" + menuIcon: "fa-solid fa-sitemap" + Applications: + displayName: "Applications" + menuIcon: "fa-solid fa-code" + Workflows: + displayName: "Workflows" + menuIcon: "fa-solid fa-route" + kadServicesInfo: + hive-metastore: + icon: "okdp-notext.svg" + menuIcon: "fa-solid fa-book" + description: | + The central repository of metadata for Hive tables and partitions, providing clients including Hive, Impala, and Spark access through the metastore service API. + A fundamental building block for modern data lakes. + home: "https://hive.apache.org" + polaris: + icon: "okdp-notext.svg" + menuIcon: "fa-solid fa-book" + description: | + Apache Polaris is an open-source, fully-featured catalog for Apache Iceberg™. + It implements Iceberg's REST API, enabling seamless multi-engine interoperability across a wide range of platforms, + including Apache Doris™, Apache Flink®, Apache Spark™, Dremio® OSS, StarRocks, and Trino. + home: "https://polaris.apache.org" + okdp-examples: + icon: "okdp-notext.svg" + menuIcon: "fa-solid fa-code" + description: | + A collection of hands-on examples, helper utilities, Jupyter notebooks, and data workflows + that showcases how to work with the OKDP Platform. + home: "https://github.com/okdp/okdp-examples" + airflow: + icon: "okdp-notext.svg" + menuIcon: "fa-solid fa-route" + description: | + Apache Airflow is a platform to programmatically author, schedule, and monitor workflows. + Use Airflow to orchestrate complex data pipelines and Spark jobs on OKDP. + home: "https://airflow.apache.org" + seaweedfs: + description: | + SeaweedFS is a simple and highly scalable distributed file system. There are the objectives to store billions of files and to serve the files fast! + home: "https://seaweedfs.com/" diff --git a/clusters/sandbox/contexts/20-provider-context.yaml b/clusters/sandbox/contexts/20-provider-context.yaml new file mode 100644 index 0000000..e3a35a9 --- /dev/null +++ b/clusters/sandbox/contexts/20-provider-context.yaml @@ -0,0 +1,57 @@ +# +# Copyright 2026 The OKDP Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +apiVersion: kubocd.kubotal.io/v1alpha1 +kind: Context +metadata: + name: provider + namespace: kubocd-system +spec: + # -- Provider interface bindings used by service contexts. Replace these with external providers that satisfy the same contracts. + context: + # -- Object storage provider contract. The sandbox binds it to SeaweedFS using S3-compatible API, STS, and console endpoints. + defaultStorage: + provider: + type: s3 + name: SeaweedFS + region: dummy + endpoints: + apiUrl: https://seaweedfs-{{ .Release.namespace }}.{{ .Context.ingress.suffix }} + stsUrl: https://seaweedfs-{{ .Release.namespace }}.{{ .Context.ingress.suffix }} + consoleUrl: https://seaweedfs-console-{{ .Release.namespace }}.{{ .Context.ingress.suffix }} + + # -- OIDC identity provider contract. The sandbox binds it to Keycloak, but any OIDC provider exposing these endpoints can be plugged in. + defaultIdp: + provider: + type: oidc + name: keycloak + issuerUri: "https://keycloak.{{ .Context.ingress.suffix }}/realms/master" + baseUri: "https://keycloak.{{ .Context.ingress.suffix }}/realms/master/protocol/openid-connect" + endpoints: + authUrl: "https://keycloak.{{ .Context.ingress.suffix }}/realms/master/protocol/openid-connect/auth" + tokenUrl: "https://keycloak.{{ .Context.ingress.suffix }}/realms/master/protocol/openid-connect/token" + userinfoUrl: "https://keycloak.{{ .Context.ingress.suffix }}/realms/master/protocol/openid-connect/userinfo" + jwksUri: "https://keycloak.{{ .Context.ingress.suffix }}/realms/master/protocol/openid-connect/certs" + + # -- Database server provider contract for services that need PostgreSQL-compatible databases. + defaultDatabaseServer: + provider: + type: postgresql + driver: org.postgresql.Driver + endpoints: + host: postgresql-instance-rw.cnpg-system.svc.cluster.local + port: 5432 + jdbcUrl: jdbc:postgresql://{{ connection.host }}:{{ connection.port }}/{{ dbName }} diff --git a/clusters/sandbox/contexts/30-service-context.yaml b/clusters/sandbox/contexts/30-service-context.yaml new file mode 100644 index 0000000..04d9879 --- /dev/null +++ b/clusters/sandbox/contexts/30-service-context.yaml @@ -0,0 +1,329 @@ +# +# Copyright 2026 The OKDP Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +apiVersion: kubocd.kubotal.io/v1alpha1 +kind: Context +metadata: + name: service + namespace: kubocd-system +spec: + # -- Base service contexts that consume provider interfaces and expose service endpoints for packages. + context: + # -- Hive Metastore service configuration for metadata storage, S3 warehouse access, and Thrift endpoint discovery. + hiveMetastore: + database: + provider: defaultDatabaseServer + name: hms + credentialsSecret: + name: creds-hive-metastore-db + usernameKey: username + passwordKey: password + storage: + provider: defaultStorage + location: + name: hive + uri: s3://hive + warehouseDir: hive-warehouse/ + credentialsSecret: + name: creds-hive-metastore-s3 + accessKeyKey: accessKey + secretKeyKey: secretKey + endpoints: + url: thrift://hive-metastore.{{ .Release.namespace }}.svc.cluster.local:9083 + + # -- Trino service configuration for catalog wiring, OIDC login, internal security, and external endpoint discovery. + trino: + catalogs: [] + oidc: + enabled: true + provider: defaultIdp + scope: "openid profile email groups" + credentialsSecret: + name: creds-trino-oauth2 + clientIdKey: client_id + clientSecretKey: client_secret + advanced: + config: + security: + # https://trino.io/docs/current/security/internal-communication.html + credentialsSecret: + name: creds-trino-security + sharedSecret: shared-secret + endpoints: + url: https://trino-{{ .Release.namespace }}.{{ .Context.ingress.suffix }} + uri: "trino://trino@trino-{{ .Release.namespace }}.{{ .Context.ingress.suffix }}" + + # -- Superset service configuration for its metadata database, examples database, Redis, OIDC, Trino integration, and endpoint. + superset: + dataSources: | + database: + provider: defaultDatabaseServer + name: superset + credentialsSecret: + name: creds-superset-db + usernameKey: username + passwordKey: password + # Secret Key for Superset's Flask App + advanced: + config: + security: + credentialsSecret: + name: creds-superset-secret-key + key: superset_secret_key + load_examples: + database: + provider: defaultDatabaseServer + connection: + jdbcUrl: "postgresql+psycopg2://{{ username }}:{{ password }}@{{ connection.host }}:{{ connection.port }}/{{ dbName }}?options=-csearch_path%3Dmain" + name: superset-examples + credentialsSecret: + name: creds-superset-examples-db + usernameKey: username + passwordKey: password + redis: + host: ~ + port: 6379 + credentialsSecret: + name: creds-superset-redis + usernameKey: username + passwordKey: password + oidc: + enabled: true + provider: defaultIdp + scope: "openid profile email groups" + usePKCE: true + credentialsSecret: + name: creds-superset-oauth2 + clientIdKey: client_id + clientSecretKey: client_secret + trino: + oidc: + enabled: true + provider: defaultIdp + scope: "openid profile email groups" + credentialsSecret: + name: creds-superset-oauth2-trino + clientIdKey: client_id + clientSecretKey: client_secret + endpoints: + url: https://superset-{{ .Release.namespace }}.{{ .Context.ingress.suffix }} + + # -- JupyterHub service configuration for the file browser, PySpark connections, OIDC, and endpoint. + jupyterHub: + welcomeNotebook: | + fileBrowser: + provider: defaultStorage + locations: [] + credentialsSecret: + name: creds-jupyterhub-s3 + accessKeyKey: S3_ACCESS_KEY + secretKeyKey: S3_SECRET_KEY + pyspark: [] + oidc: + enabled: true + provider: defaultIdp + credentialsSecret: + name: creds-jupyterhub-oauth2 + clientIdKey: client_id + clientSecretKey: client_secret + jupyterHubCryptKey: JUPYTERHUB_CRYPT_KEY + endpoints: + url: https://jupyterhub-{{ .Release.namespace }}.{{ .Context.ingress.suffix }} + + # -- Spark runtime defaults and Kubernetes RBAC settings shared by Spark applications. + spark: + rbac: + rbac: + create: true + serviceAccount: + create: true + name: "spark" + automountServiceAccountToken: true + defaults: + spark.eventLog.dir: "s3a://spark-events/event-logs/" + spark.eventLog.enabled: true + spark.eventLog.compress: true + spark.eventLog.rolling.enabled: true + spark.hive.metastore.uris: "{{ hiveMetastore.endpoints.url }}" + spark.hive.metastore.client.connect.retry.delay: 5 + spark.hive.metastore.client.socket.timeout: 1800 + spark.sql.debug.maxToStringFields: 25 + spark.network.timeout: 600 + + # -- Spark History Server configuration for event log storage, retention, OIDC protection, and endpoint. + sparkHistory: + storage: + provider: defaultStorage + credentialsSecret: + name: creds-spark-history-s3 + accessKeyKey: accessKey + secretKeyKey: secretKey + config: + spark.history.fs.logDirectory: "s3a://spark-events/event-logs/" + spark.history.kerberos.enabled: false + spark.history.ui.acls.enable: true + spark.history.fs.eventLog.rolling.maxFilesToRetain: 5 + spark.history.fs.cleaner.enabled: true + spark.history.fs.cleaner.interval: 1d + spark.history.fs.cleaner.maxAge: 7d + spark.history.fs.update.interval: 60s + spark.hadoop.fs.s3a.connection.ssl.enabled: true + spark.hadoop.fs.s3a.impl: org.apache.hadoop.fs.s3a.S3AFileSystem + spark.hadoop.fs.s3a.path.style.access: true + spark.ui.filters: io.okdp.spark.authc.OidcAuthFilter + spark.user.groups.mapping: io.okdp.spark.authz.OidcGroupMappingServiceProvider + oidc: + enabled: true + provider: defaultIdp + scope: "openid+profile+email+offline_access+groups" + usePKCE: true + credentialsSecret: + name: creds-spark-history-oauth2 + clientIdKey: client_id + clientSecretKey: client_secret + endpoints: + url: https://spark-history-{{ .Release.namespace }}.{{ .Context.ingress.suffix }} + + # -- Polaris service configuration for realms, database, storage, OIDC principal mapping, features, CORS, and endpoint. + polaris: + realms: [] + database: + provider: defaultDatabaseServer + name: polaris + credentialsSecret: + name: creds-polaris-db + usernameKey: username + passwordKey: password + storage: + provider: defaultStorage + # The access key is used for STS interactions between Polaris and S3 (sts:AssumeRole) + # + catalog-side Iceberg metadata ops + # The Polaris SeaweedFS identity for STS + credentialsSecret: + name: creds-polaris-s3 + accessKeyKey: accessKey + secretKeyKey: secretKey + oidc: + enabled: true + provider: defaultIdp + scope: "openid+profile+email+offline_access+groups" + credentialsSecret: + name: creds-polaris-oauth2 + clientIdKey: client_id + clientSecretKey: client_secret + # Map polaris internal principal and roles with OIDC principal and roles + # Here, the oidc principal should exist in the OIDC provider as service-account-svc-polaris-api-admin having roles: + # polaris_catalog_admin and polaris_service_admin + principalMapper: + # service-account-svc-polaris-api-admin oidc principal maps to polaris service-account-svc-polaris-api-admin user + type: default + # You can use 'sub' as stable and immutable identity + idClaimPath: ~ + nameClaimPath: "preferred_username" + # Issue: https://github.com/apache/polaris/issues/2373 + # Polaris activates only the internal principal roles that are actually granted to that principal in Polaris RBAC + # Active Polaris roles = (mapped external roles) ∩ (principal roles granted in Polaris) + principalRolesMapper: + type: default + rolesClaimPath: realm_access/roles + filter: "^(?!default-roles-.*|offline_access$|uma_authorization$).*$" + mappings: + # The rules are applied sequentially as ordered transforms + # Strip the "polaris_" prefix from admin roles and convert them to Polaris roles: + # polaris_service_admin -> PRINCIPAL_ROLE:service_admin + # polaris_catalog_admin -> PRINCIPAL_ROLE:catalog_admin + # platform_admin -> PRINCIPAL_ROLE:platform_admin + - regex: "^polaris_(.*)$" + replacement: "PRINCIPAL_ROLE:$1" + # Fallback for normal roles, but do NOT touch values already prefixed by Polaris. + - regex: "^(?!PRINCIPAL_ROLE:)(?!polaris_)(.*)$" + replacement: "PRINCIPAL_ROLE:$1" + + features: + DROP_WITH_PURGE_ENABLED: true + # ENFORCE_PRINCIPAL_CREDENTIAL_ROTATION_REQUIRED_CHECKING: false + SUPPORTED_CATALOG_STORAGE_TYPES: + - S3 + realmOverrides: {} + # my-realm: + # ENFORCE_PRINCIPAL_CREDENTIAL_ROTATION_REQUIRED_CHECKING: true + advanced: + config: + polaris.realm-context.header-name: "Polaris-Realm" + # Spark and Trino does not pass the header + # polaris.realm-context.require-header: false + quarkus.http.cors.enabled: "true" + cors: + allowedOrigins: + - "https://polaris-console-{{ .Release.namespace }}.{{ .Context.ingress.suffix }}" + allowedMethods: + - "GET" + - "POST" + - "PUT" + - "DELETE" + - "PATCH" + - "OPTIONS" + allowedHeaders: + - "Content-Type" + - "Authorization" + - "Polaris-Realm" + - "X-Request-ID" + exposedHeaders: + - "*" + accessControlMaxAge: "PT10M" + accessControlAllowCredentials: true + endpoints: + url: https://polaris-{{ .Release.namespace }}.{{ .Context.ingress.suffix }} + + # -- Polaris Console configuration for default realm selection, API endpoint, OIDC client, and console endpoint. + polarisConsole: + defaultRealm: ~ + apiUrl: https://polaris-{{ .Release.namespace }}.{{ .Context.ingress.suffix }} + oidc: + enabled: true + provider: defaultIdp + scope: "openid profile email roles offline_access" + # Public OIDC client with PKCE enabled + clientId: polaris-console + endpoints: + url: https://polaris-console-{{ .Release.namespace }}.{{ .Context.ingress.suffix }} + + airflow: + database: + provider: defaultDatabaseServer + name: airflow + credentialsSecret: + name: creds-airflow-db + connectionKey: connection + oidc: + enabled: true + provider: defaultIdp + scope: "openid+profile+email+offline_access+groups" + credentialsSecret: + name: creds-airflow-oauth2 + clientIdKey: client_id + clientSecretKey: client_secret + advanced: + config: + security: + credentialsSecret: + name: creds-airflow-security + jwtSecret: jwt-secret + apiSecretKey: api-secret-key + fernetKey: fernet-key + endpoint: + url: https://airflow-{{ .Release.namespace }}.{{ .Context.ingress.suffix }} + diff --git a/clusters/sandbox/contexts/99-examples-context.yaml b/clusters/sandbox/contexts/99-examples-context.yaml new file mode 100644 index 0000000..e79f8d4 --- /dev/null +++ b/clusters/sandbox/contexts/99-examples-context.yaml @@ -0,0 +1,1169 @@ +# +# Copyright 2026 The OKDP Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +apiVersion: kubocd.kubotal.io/v1alpha1 +kind: Context +metadata: + name: examples + namespace: kubocd-system +spec: + # -- Example data model and sandbox-specific configuration layered on top of the base providers and services. + context: + # -- Example Trino catalogs and connector definitions for benchmark data and the bronze/silver/gold lakehouse model. + trino: + # -- Names of `trino.connections` to materialize as Trino catalogs. Each entry must match a connection name below. + catalogs: + - tpch + - tpcds + - bronze + - silver + - gold + # -- Reusable Trino catalog connection specs. Referenced entries are rendered as connector properties for the active catalogs. + connections: + - name: tpch + description: | + TPC-H benchmark catalog with generated decision-support data for testing analytical SQL + queries and engine performance. + properties: | + connector.name=tpch + tpch.splits-per-node=4 + - name: tpcds + description: | + TPC-DS benchmark catalog with generated retail decision-support data for testing complex analytics + workloads and query performance. + properties: | + connector.name=tpcds + tpcds.splits-per-node=4 + - name: bronze + description: | + Bronze medallion catalog for raw ingested datasets registered in Hive Metastore before cleansing + or standardization. + service: + name: hiveMetastore + storage: + provider: defaultStorage + credentialsSecret: + name: creds-trino-s3 + accessKeyKey: S3_ACCESS_KEY + secretKeyKey: S3_SECRET_ACCESS + properties: | + connector.name=hive + hive.metastore.uri={{ service.endpoints.url }} + hive.metastore.username=trino + hive.non-managed-table-writes-enabled=true + + s3.endpoint={{ storage.endpoints.apiUrl }} + s3.aws-access-key=${ENV:S3_ACCESS_KEY} + s3.aws-secret-key=${ENV:S3_SECRET_ACCESS} + fs.native-s3.enabled=true + s3.region={{ storage.region }} + s3.path-style-access=true + - name: silver + description: | + Silver medallion catalog for cleaned and conformed Iceberg datasets managed through Polaris for downstream analytics. + service: + name: polaris + oidc: + provider: defaultIdp + credentialsSecret: + name: creds-trino-oauth2-polaris + clientIdKey: client_id + clientSecretKey: client_secret + storage: + provider: defaultStorage + # Enable vended credentials - uncomment to disable + # credentialsSecret: + # name: creds-trino-s3 + # accessKeyKey: S3_ACCESS_KEY + # secretKeyKey: S3_SECRET_ACCESS + properties: | + connector.name=iceberg + iceberg.rest-catalog.warehouse={{ connection.name }} + iceberg.catalog.type=rest + iceberg.rest-catalog.uri={{ service.endpoints.url }}/api/catalog + iceberg.rest-catalog.oauth2.server-uri={{ idp.endpoints.tokenUrl }} + iceberg.rest-catalog.security=OAUTH2 + iceberg.rest-catalog.oauth2.credential=${ENV:client_id}:${ENV:client_secret} + iceberg.rest-catalog.oauth2.scope=profile + + s3.endpoint={{ storage.endpoints.apiUrl }} + fs.native-s3.enabled=true + s3.region={{ storage.region }} + s3.path-style-access=true + + # Enable vended credentials - uncomment to disable + iceberg.rest-catalog.vended-credentials-enabled=true + # s3.aws-access-key=${ENV:S3_ACCESS_KEY} + # s3.aws-secret-key=${ENV:S3_SECRET_ACCESS} + - name: gold + description: | + Gold medallion catalog for curated, business-ready Iceberg datasets optimized for reporting, BI, and shared consumption. + service: + name: polaris + oidc: + provider: defaultIdp + credentialsSecret: + name: creds-trino-oauth2-polaris + clientIdKey: client_id + clientSecretKey: client_secret + storage: + provider: defaultStorage + # Enable vended credentials - uncomment to disable + # credentialsSecret: + # name: creds-trino-s3 + # accessKeyKey: S3_ACCESS_KEY + # secretKeyKey: S3_SECRET_ACCESS + properties: | + connector.name=iceberg + iceberg.rest-catalog.warehouse={{ connection.name }} + iceberg.catalog.type=rest + iceberg.rest-catalog.uri={{ service.endpoints.url }}/api/catalog + iceberg.rest-catalog.oauth2.server-uri={{ idp.endpoints.tokenUrl }} + iceberg.rest-catalog.security=OAUTH2 + iceberg.rest-catalog.oauth2.credential=${ENV:client_id}:${ENV:client_secret} + iceberg.rest-catalog.oauth2.scope=profile + + s3.endpoint={{ storage.endpoints.apiUrl }} + fs.native-s3.enabled=true + s3.region={{ storage.region }} + s3.path-style-access=true + + # Enable vended credentials - uncomment to disable + iceberg.rest-catalog.vended-credentials-enabled=true + # s3.aws-access-key=${ENV:S3_ACCESS_KEY} + # s3.aws-secret-key=${ENV:S3_SECRET_ACCESS} + + # -- Example Superset data sources that connect to the bronze, silver, and gold Trino catalogs. + superset: + # -- Superset data source selector. The `databases` list references names from `superset.connections` to import as Superset databases. + dataSources: | + databases: + - bronze + - silver + - gold + # -- Reusable Superset database connection specs. Referenced entries render the SQLAlchemy/YAML database definitions used by Superset. + connections: + - name: bronze + description: | + Superset connection to the Bronze Trino catalog for exploring raw ingested datasets before cleansing + and standardization. + service: + name: trino + yaml: | + database_name: hms-bronze + sqlalchemy_uri: "{{ service.endpoints.uri }}:443/bronze?http_scheme=https&verify=false" + impersonate_user: true + extra: | + { + "engine_params": {}, + "metadata_params": {}, + "connect_args": { + "http_scheme": "https", + "verify": false + }, + "metadata_cache_timeout": {}, + "schemas_allowed_for_file_upload": [] + } + allow_run_async: true + allow_dml: true + allow_ctas: true + allow_cvas: true + expose_in_sqllab: true + - name: silver + description: | + Superset connection to the Silver Trino catalog for analyzing cleaned and conformed datasets prepared for + downstream analytics. + service: + name: trino + yaml: | + database_name: polaris-silver + sqlalchemy_uri: "{{ service.endpoints.uri }}:443/silver?http_scheme=https&verify=false" + impersonate_user: true + extra: | + { + "engine_params": {}, + "metadata_params": {}, + "connect_args": { + "http_scheme": "https", + "verify": false + }, + "metadata_cache_timeout": {}, + "schemas_allowed_for_file_upload": [] + } + allow_run_async: true + allow_dml: true + allow_ctas: true + allow_cvas: true + expose_in_sqllab: true + - name: gold + description: | + Superset connection to the Gold Trino catalog for reporting on curated, business-ready datasets optimized for dashboards and BI. + service: + name: trino + yaml: | + database_name: polaris-gold + sqlalchemy_uri: "{{ service.endpoints.uri }}:443/gold?http_scheme=https&verify=false" + impersonate_user: true + extra: | + { + "engine_params": {}, + "metadata_params": {}, + "connect_args": { + "http_scheme": "https", + "verify": false + }, + "metadata_cache_timeout": {}, + "schemas_allowed_for_file_upload": [] + } + allow_run_async: true + allow_dml: true + allow_ctas: true + allow_cvas: true + expose_in_sqllab: true + oidc: + roleMapping: + "platform_admin": ["Admin"] + "data_engineer": ["Alpha", "sql_lab"] + "data_scientist": ["Alpha", "sql_lab"] + "business_analyst": ["Gamma"] + "data_steward": ["Gamma", "sql_lab"] + "auditor": ["Gamma"] + + # -- Example Spark History Server OIDC role mapping + sparkHistory: + oidc: + roleMapping: + # spark.admin.acls.groups + admin_groups: + - platform_admin + # spark.history.ui.admin.acls.groups + history_admin_groups: + - platform_admin + - auditor + # spark.modify.acls.groups + modify_groups: + - platform_admin + - data_engineer + # spark.ui.view.acls.groups + view_groups: + - platform_admin + - data_engineer + - data_scientist + - data_steward + - business_analyst + + # -- Example Polaris realm model, administrative principals, catalogs, catalog roles, principal roles, and demo principal grants. + polaris: + realms: + - name: sandbox + # Polaris built-in bootstrap 'root' principal for this realm. + # Used only by svc-polaris-api-admin-tool to set the credentials CLIENT_ID/CLIENT_SECRET + # (stored as hashes in the metastore DB). Do NOT map normal OIDC logins to 'root' principal. + # Docs: https://polaris.apache.org/releases/1.3.0/configuring-polaris-for-production/#bootstrapping + rootPrincipal: + name: root + credentialsSecret: + name: creds-polaris-root-okdp-sandbox + clientIdKey: client_id + clientSecretKey: client_secret + # External OIDC-based administrative principals (non root, limited permissions) + # Service admin: manages principals/principal-roles and server-level admin operations. + # Catalog admin: manages catalog roles, grants, and catalog-level access control. + principals: + # - In production, split into separate principals (least privilege), e.g.: + # * polaris_service_admin -> service_admin (principals/principal-roles, server/realm admin) + # * polaris_catalog_admin -> catalog_admin (catalog roles, grants, catalog access control) + # - Create additional principals for specific purposes (e.g., CI/CD, automation, etc.) with only the + # roles they need. + # + # - For the Sandbox env, you may combine multiple admin roles into a single principal for simplicity. + # Service Account Polaris Admin (for polaris automation) + - name: service-account-svc-polaris-api-admin + clientId: svc-polaris-api-admin + credentialsSecret: + name: creds-polaris-oauth2-admin-okdp-sandbox + clientIdKey: client_id + clientSecretKey: client_secret + roles: + - service_admin + - catalog_admin + # Interactive user Polaris Administrator (for polaris console) + - name: adm + roles: + - service_admin + - catalog_admin + + # -- Example Polaris Console defaults for the sandbox realm. + polarisConsole: + defaultRealm: sandbox + + # -- Example JupyterHub file browser locations, Spark catalog connections, and welcome notebook for the demo lakehouse. + jupyterHub: + fileBrowser: + locations: + - name: bronze + uri: s3://bronze + - name: silver + uri: s3://silver + - name: gold + uri: s3://gold + # -- Names of `jupyterHub.connections` to inject into the PySpark runtime. Each entry contributes Spark properties for notebooks. + pyspark: + - lakehouse + - silver + - gold + # -- Reusable JupyterHub connection specs. `pyspark` references these by name to configure S3 access and Iceberg/Polaris Spark catalogs. + connections: + - name: lakehouse + description: | + Spark object storage connection for reading and writing lakehouse datasets directly in the S3 + from Jupyter notebooks. + storage: + provider: defaultStorage + credentialsSecret: + name: creds-examples-s3 + accessKeyKey: S3_ACCESS_KEY + secretKeyKey: S3_SECRET_KEY + properties: | + spark.hadoop.fs.s3a.endpoint={{ storage.endpoints.apiUrl }} + spark.hadoop.fs.s3a.impl=org.apache.hadoop.fs.s3a.S3AFileSystem + spark.hadoop.fs.s3a.aws.credentials.provider=org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider + spark.hadoop.fs.s3a.access.key=$(S3_ACCESS_KEY) + spark.hadoop.fs.s3a.secret.key=$(S3_SECRET_KEY) + spark.hadoop.fs.s3a.path.style.access=true + spark.hadoop.fs.s3a.connection.maximum=100 + spark.hadoop.fs.s3a.connection.ssl.enabled=true + spark.hadoop.fs.s3a.endpoints.region={{ storage.region }} + spark.sql.extensions=org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions + - name: silver + description: | + Spark Iceberg REST catalog connection for exploring and transforming cleaned, conformed Silver datasets + from Jupyter notebooks through Polaris. + service: + name: polaris + storage: + provider: defaultStorage + properties: | + spark.sql.extensions=org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions + spark.sql.catalog.{{ connection.name }}.header.Polaris-Realm=sandbox + spark.sql.catalog.{{ connection.name }}.type=rest + spark.sql.catalog.{{ connection.name }}=org.apache.iceberg.spark.SparkCatalog + spark.sql.catalog.{{ connection.name }}.warehouse={{ connection.name }} + spark.sql.catalog.{{ connection.name }}.uri={{ service.endpoints.url }}/api/catalog + spark.sql.catalog.{{ connection.name }}.oauth2-server-uri={{ idp.endpoints.tokenUrl }} + spark.sql.catalog.{{ connection.name }}.scope=profile + spark.sql.catalog.{{ connection.name }}.header.X-Iceberg-Access-Delegation=vended-credentials + spark.sql.catalog.{{ connection.name }}.rest.auth.type=oauth2 + spark.sql.catalog.{{ connection.name }}.token-refresh-enabled=true + spark.sql.catalog.{{ connection.name }}.client.region={{ storage.region }} + spark.sql.catalog.{{ connection.name }}.s3.region={{ storage.region }} + spark.sql.catalog.{{ connection.name }}.io-impl=org.apache.iceberg.io.ResolvingFileIO + - name: gold + description: | + Spark Iceberg REST catalog connection for querying and producing curated, business-ready Gold datasets + from Jupyter notebooks through Polaris. + service: + name: polaris + storage: + provider: defaultStorage + properties: | + spark.sql.extensions=org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions + spark.sql.catalog.{{ connection.name }}.header.Polaris-Realm=sandbox + spark.sql.catalog.{{ connection.name }}.type=rest + spark.sql.catalog.{{ connection.name }}=org.apache.iceberg.spark.SparkCatalog + spark.sql.catalog.{{ connection.name }}.warehouse={{ connection.name }} + spark.sql.catalog.{{ connection.name }}.uri={{ service.endpoints.url }}/api/catalog + spark.sql.catalog.{{ connection.name }}.oauth2-server-uri={{ idp.endpoints.tokenUrl }} + spark.sql.catalog.{{ connection.name }}.scope=profile + spark.sql.catalog.{{ connection.name }}.header.X-Iceberg-Access-Delegation=vended-credentials + spark.sql.catalog.{{ connection.name }}.rest.auth.type=oauth2 + spark.sql.catalog.{{ connection.name }}.token-refresh-enabled=true + spark.sql.catalog.{{ connection.name }}.client.region={{ storage.region }} + spark.sql.catalog.{{ connection.name }}.s3.region={{ storage.region }} + spark.sql.catalog.{{ connection.name }}.io-impl=org.apache.iceberg.io.ResolvingFileIO + oidc: + roleMapping: + admin_groups: + - platform_admin + allowed_groups: + - platform_admin + - data_engineer + - data_scientist + - data_steward + - business_analyst + welcomeNotebook: | + { + "cells": [ + { + "cell_type": "markdown", + "id": "127ae6f0", + "metadata": {}, + "source": [ + "# 👋 Welcome to OKDP Examples\n", + "\n", + "This workspace contains hands-on examples and ready-to-run workflows for the **[Open Kubernetes Data Platform (OKDP)](https://okdp.io)**.\n", + "\n", + "You’ll be able to explore:\n", + "\n", + "- **[SeaweedFS](https://github.com/seaweedfs/seaweedfs)** for S3-compatible object storage\n", + "- **[Hive Metastore](https://hive.apache.org/)** for table/catalog metadata\n", + "- **[Trino](https://trino.io/)** for interactive SQL queries\n", + "- **[Jupyter](https://jupyter.org/)** for notebooks and analysis\n", + "- **[Apache Superset](https://superset.apache.org/)** for dashboards and visualizations\n", + "\n", + "---\n", + "\n", + "## 🚀 Get started\n", + "\n", + "Click the link below to **download or update** the [okdp-examples](https://github.com/OKDP/okdp-examples) repository directly into your JupyterLab workspace.\n", + "\n", + "> Tip: If you already cloned it before, this will **pull the latest changes**.\n", + "\n", + "[✅ Sync OKDP Examples]({{ $jupyterUrl }}/hub/user-redirect/git-pull?repo=https://github.com/OKDP/okdp-examples&branch=main&urlpath=lab/tree/okdp-examples/notebooks/index.ipynb)\n", + "\n", + "---\n", + "\n", + "## 🧭 What’s inside the repo\n", + "\n", + "- **Helm-driven workflows** to download public datasets and upload them to S3 (SeaweedFS)\n", + "- **Trino SQL** examples to create external tables and sync partitions\n", + "- **Jupyter notebooks** to query Trino (SQLAlchemy + native SQL)\n", + "- Optional **Superset** workflows for visualization\n", + "\n", + "If you get stuck, open the repo’s [README](https://github.com/OKDP/okdp-examples) and follow the step-by-step instructions.\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.12" + } + }, + "nbformat": 4, + "nbformat_minor": 5 + } + + airflow: + dags: + gitSync: + enabled: true + repo: https://github.com/OKDP/okdp-examples.git + branch: main + ref: HEAD + depth: 1 + subPath: airflow/dags + period: 60s + # credentialsSecret: + # name: creds-git-examples + # usernameKey: GITSYNC_USERNAME + # passwordKey: GITSYNC_PASSWORD + oidc: + # https://airflow.apache.org/docs/apache-airflow-providers-fab/stable/auth-manager/access-control.html + # Map OIDC roles into airflow internal roles + roleMapping: + platform_admin: [ "Admin" ] + data_engineer: [ "Op" ] + data_scientist: [ "User" ] + business_analyst: [ "Viewer" ] + data_steward: [ "Viewer" ] + auditor: [ "Viewer" ] + + examples: + mylakehouse: + bronze: + storageUri: s3://bronze + prefix: mobility/nyc_tlc + dataDownloadUrl: https://d37ci6vzurychx.cloudfront.net/trip-data + silver: + storageUri: s3://silver + gold: + storageUri: s3://gold + storage: + provider: defaultStorage + credentialsSecret: + name: creds-examples-s3 + accessKeyKey: S3_ACCESS_KEY + secretKeyKey: S3_SECRET_KEY + trino: + url: https://trino-{{ .Release.namespace }}.{{ .Context.platform.ingress.suffix }} + oidc: + enabled: true + provider: defaultIdp + credentialsSecret: + name: creds-examples-oauth2-trino + clientIdKey: client_id + clientSecretKey: client_secret + polaris: + realms: + - name: sandbox + catalogs: + - name: silver + type: INTERNAL + properties: + default-base-location: s3://silver/ + storageConfigInfo: + storageType: S3 + allowedLocations: + - "s3://silver/" + endpoint: "$S3_ENDPOINT" + endpointInternal: "$S3_ENDPOINT" + stsEndpoint: "$S3_ENDPOINT" + pathStyleAccess: true + stsUnavailable: false + region: "dummy" + roleArn: "arn:aws:iam::000000000000:role/S3WriteRole" + + - name: gold + type: INTERNAL + properties: + default-base-location: s3://gold/ + storageConfigInfo: + storageType: S3 + allowedLocations: + - "s3://gold/" + endpoint: "$S3_ENDPOINT" + endpointInternal: "$S3_ENDPOINT" + stsEndpoint: "$S3_ENDPOINT" + pathStyleAccess: true + stsUnavailable: false + region: "dummy" + roleArn: "arn:aws:iam::000000000000:role/S3WriteRole" + catalogRoles: + # Polaris example roles: + # - Catalog readers: read-only access to tables in the catalog + # - Catalog contributor: read and write access to tables in the catalog + # In the Polaris RBAC example, Catalog contributor also creates namespaces and tables. + # Since this deployment example grants privileges only at catalog level, these privileges + # apply across the whole catalog. + # https://polaris.apache.org/in-dev/unreleased/managing-security/access-control + - name: catalog_contributor + grants: + - onCatalogs: [silver, gold] + privileges: + - CATALOG_READ_PROPERTIES + - NAMESPACE_CREATE + - NAMESPACE_LIST + - NAMESPACE_READ_PROPERTIES + - TABLE_CREATE + - TABLE_LIST + - TABLE_READ_PROPERTIES + - TABLE_WRITE_PROPERTIES + - TABLE_READ_DATA + - TABLE_WRITE_DATA + - VIEW_CREATE + - VIEW_LIST + - VIEW_READ_PROPERTIES + - VIEW_WRITE_PROPERTIES + + - name: catalog_reader + grants: + - onCatalogs: [silver, gold] + privileges: + - CATALOG_READ_PROPERTIES + - NAMESPACE_LIST + - NAMESPACE_READ_PROPERTIES + - TABLE_LIST + - TABLE_READ_PROPERTIES + - TABLE_READ_DATA + - VIEW_LIST + - VIEW_READ_PROPERTIES + + - name: data_administrator + grants: + - onCatalogs: [silver, gold] + privileges: + - CATALOG_READ_PROPERTIES + - CATALOG_WRITE_PROPERTIES + # namespace administration + - NAMESPACE_CREATE + - NAMESPACE_LIST + - NAMESPACE_READ_PROPERTIES + - NAMESPACE_WRITE_PROPERTIES + - NAMESPACE_DROP + # metadata administration on tables/views + - TABLE_LIST + - TABLE_READ_PROPERTIES + - TABLE_WRITE_PROPERTIES + - TABLE_DROP + - VIEW_LIST + - VIEW_READ_PROPERTIES + - VIEW_WRITE_PROPERTIES + - VIEW_DROP + # policy lifecycle + - POLICY_CREATE + - POLICY_READ + - POLICY_WRITE + - POLICY_LIST + - POLICY_DROP + - CATALOG_ATTACH_POLICY + - CATALOG_DETACH_POLICY + + principalRoles: + + - name: business_analyst + catalogRoles: + - catalog_reader + + - name: data_scientist + catalogRoles: + - catalog_reader + + - name: data_steward + catalogRoles: + - catalog_reader + - data_administrator + + - name: data_engineer + catalogRoles: + - catalog_contributor + - data_administrator + + - name: auditor + catalogRoles: + - catalog_reader + + - name: platform_admin + catalogRoles: + - catalog_contributor + - data_administrator + + # Polaris management/admin principal roles used for admin-plane access + # (for example via OIDC role mapping such as PRINCIPAL_ROLE:service_admin + # and PRINCIPAL_ROLE:catalog_admin). Keep them declared in Polaris even if + # they have no catalogRoles here, because Polaris requires principal roles + # referenced by tokens/principals to already exist. These are not regular + # catalog data-access roles like data_engineer or data_scientist. + - name: service_admin + catalogRoles: [] + + - name: catalog_admin + catalogRoles: [] + + # https://github.com/apache/polaris/issues/2373 + principals: + - name: bob + principalRoles: + - data_engineer + + - name: mark + principalRoles: + - data_scientist + + - name: nina + principalRoles: + - business_analyst + + - name: grace + principalRoles: + - data_steward + + - name: alice + principalRoles: + - platform_admin + - service_admin + - catalog_admin + + - name: eve + principalRoles: + - auditor + + - name: adm + principalRoles: + - platform_admin + - service_admin + - catalog_admin + + - name: service-account-svc-trino-polaris-writer + principalRoles: + - platform_admin + - service_admin + - catalog_admin + + - name: service-account-svc-spark-etl-polaris-writer + principalRoles: + - data_engineer + - data_scientist + - business_analyst + + - name: service-account-svc-polaris-api-admin + principalRoles: + - service_admin + - catalog_admin + + # -- Sandbox-specific storage provider implementation settings for SeaweedFS databases, S3 credentials, STS, and identities. + defaultStorage: + # Sandbox implementation-specific + # Consider this part already provided by your platform + # settings: {} + settings: + filer: + database: + provider: defaultDatabaseServer + name: seaweedfs + credentialsSecret: + name: creds-seaweedfs-db + usernameKey: username + passwordKey: password + s3: + enabled: true + s3_secret: creds-seaweedfs-identities + bucketAnonymousRead: true + credentialsSecret: + name: creds-seaweedfs-s3 + accessKeyKey: accessKey + secretKeyKey: secretKey + + sts: + enabled: true + roleArn: arn:aws:iam::000000000000:role/S3WriteRole + + advanced: + identities: + hiveMetastore: + identity: hiveMetastore + actions: [ Read, Write, List ] + trino: + identity: trino + actions: [ Read, Write, List, Admin ] + jupyterHub: + identity: jupyterHub + actions: [ Read, Write, List ] + sparkHistory: + identity: sparkHistory + actions: [ Read, Write, List ] + # SeaweedFS matches the accessKey above in the SigV4 request to the configured S3 credentials + # under this identity. The identity name can be anything and has importance for debugging issues + # SeaweedFS expects an admin identity to authorize it for sts:AssumeRole. + # response: code=AccessDenied, type=Sender, message=user + # polaris is not authorized to assume role arn:aws:iam::000000000000:role/S3WriteRole + polaris: + identity: polaris + actions: [ Read, Write, List, Admin ] + airflow: + identity: airflow + actions: [ Read, Write, List, Admin ] + examples: + identity: examples + actions: [ Admin, Read, Write, List ] + seaweedfs: + identity: seaweedfs + actions: [ Admin ] + + # -- Sandbox-specific identity provider settings for Keycloak users, roles, clients, service accounts, and OIDC scopes. + defaultIdp: + # Sandbox implementation-specific + # Consider this part already provided by your IDP + # settings: {} + settings: + database: + provider: defaultDatabaseServer + name: keycloak + credentialsSecret: + name: creds-keycloak-db + usernameKey: username + passwordKey: password + realm: + name: master + displayName: "OKDP internal Realm" + accessTokenLifespan: 3600 + users: + - username: bob + email: bob@okdp.io + firstName: bob + lastName: bob + password: bob + roles: [data_engineer] + - username: mark + email: mark@okdp.io + firstName: mark + lastName: mark + password: mark + roles: [data_scientist] + - username: nina + email: nina@okdp.io + firstName: nina + lastName: nina + password: nina + roles: [business_analyst] + - username: grace + email: grace@okdp.io + firstName: grace + lastName: grace + password: grace + roles: [data_steward] + - username: alice + email: alice@okdp.io + firstName: alice + lastName: alice + password: alice + roles: [platform_admin, polaris_service_admin, polaris_catalog_admin] + - username: eve + email: eve@okdp.io + firstName: eve + lastName: eve + password: eve + roles: [auditor] + - username: adm + email: adm@okdp.io + firstName: adm + lastName: adm + password: adm + roles: [ platform_admin, polaris_service_admin, polaris_catalog_admin ] + - username: svc-spark-etl-polaris-writer + serviceAccountClientId: svc-spark-etl-polaris-writer + roles: [ data_engineer, data_scientist, business_analyst ] + # Add the polaris service account user + # Here the same service account has service_admin and catalog_admin in Polaris + # The service account behaves like a super admin, you can split as your convenience + - username: svc-polaris-api-admin + serviceAccountClientId: svc-polaris-api-admin + roles: [ polaris_service_admin, polaris_catalog_admin ] + - username: svc-trino-polaris-writer + serviceAccountClientId: svc-trino-polaris-writer + roles: [ platform_admin, polaris_service_admin, polaris_catalog_admin ] + roles: + - name: data_engineer + - name: data_scientist + - name: business_analyst + - name: platform_admin + - name: data_steward + - name: auditor + # Add the polaris management roles + # These roles can be named anyway, + # but they should map to polaris management roles (service_admin, and catalog_admin) + - name: polaris_service_admin + - name: polaris_catalog_admin + clients: + - clientId: okdp-server + name: okdp-server + publicClient: true + redirectUris: + - "https://okdp-server.{{ .Context.ingress.suffix }}/swagger/oauth2-redirect.html" + webOrigins: + - "https://okdp-server.{{ .Context.ingress.suffix }}" + - clientId: okdp-ui + name: okdp-ui + publicClient: true + redirectUris: + - "https://okdp-ui.{{ .Context.ingress.suffix }}/index.html" + - "https://okdp-ui.{{ .Context.ingress.suffix }}/silent-refresh.html" + webOrigins: + - "https://okdp-ui.{{ .Context.ingress.suffix }}" + - clientId: polaris-console + name: polaris-console + publicClient: true + redirectUris: + - "https://polaris-console-default.{{ .Context.ingress.suffix }}/auth/callback" + webOrigins: + - "https://polaris-console-default.{{ .Context.ingress.suffix }}" + - clientId: seaweedfs-s3 + name: seaweedfs-s3 + publicClient: true + - clientId: superset + name: superset + secret: mySupersetSecret + publicClient: false + redirectUris: + - "https://*" + webOrigins: + - "https://*" + defaultClientScopes: + - web-origins + - acr + - roles + - profile + - groups + - basic + - email + optionalClientScopes: + - address + - phone + - organization + - offline_access + - microprofile-jwt + - clientId: jupyterhub + name: jupyterhub + secret: myJupyterHubSecret + publicClient: false + redirectUris: + - "https://*" + webOrigins: + - "https://*" + defaultClientScopes: + - web-origins + - acr + - roles + - profile + - groups + - basic + - email + optionalClientScopes: + - address + - phone + - organization + - offline_access + - microprofile-jwt + - clientId: trino + name: trino + secret: myTrinoSecret + publicClient: false + redirectUris: + - "https://*" + webOrigins: + - "https://*" + defaultClientScopes: + - web-origins + - acr + - roles + - profile + - groups + - basic + - email + optionalClientScopes: + - address + - phone + - organization + - offline_access + - microprofile-jwt + - clientId: spark-history + name: spark-history + secret: mySparkHistorySecret + publicClient: false + redirectUris: + - "https://*" + webOrigins: + - "https://*" + defaultClientScopes: + - web-origins + - acr + - roles + - profile + - groups + - basic + - email + optionalClientScopes: + - address + - phone + - organization + - offline_access + - microprofile-jwt + - clientId: superset-trino + name: superset-trino + secret: mySupersetTrinoSecret + publicClient: false + redirectUris: + - "https://*" + webOrigins: + - "https://*" + defaultClientScopes: + - web-origins + - acr + - roles + - profile + - groups + - basic + - email + optionalClientScopes: + - address + - phone + - organization + - offline_access + - microprofile-jwt + - clientId: polaris + name: polaris + secret: myPolarisSecret + publicClient: false + # Enable service accounts on the polaris client + serviceAccountsEnabled: true + standardFlowEnabled: false + implicitFlowEnabled: false + directAccessGrantsEnabled: false + redirectUris: + - "https://*" + webOrigins: + - "https://*" + defaultClientScopes: + - web-origins + - acr + - roles + - profile + - groups + - basic + - email + optionalClientScopes: + - address + - phone + - organization + - offline_access + - microprofile-jwt + - clientId: airflow + name: airflow + secret: myAirflowSecret + publicClient: false + redirectUris: + - "https://*" + webOrigins: + - "https://*" + defaultClientScopes: + - web-origins + - acr + - roles + - profile + - groups + - basic + - email + optionalClientScopes: + - address + - phone + - organization + - offline_access + - microprofile-jwt + - clientId: svc-trino-polaris-writer + name: svc-trino-polaris-writer + secret: myTrinoPolarisSecret + publicClient: false + # Enable service accounts on the polaris client + serviceAccountsEnabled: true + standardFlowEnabled: false + implicitFlowEnabled: false + directAccessGrantsEnabled: false + defaultClientScopes: + - groups + - roles + - profile + - email + - clientId: svc-trino-examples-writer + name: svc-trino-examples-writer + secret: myTrinoExamplesSecret + publicClient: false + serviceAccountsEnabled: true + standardFlowEnabled: false + implicitFlowEnabled: false + directAccessGrantsEnabled: false + defaultClientScopes: + - groups + - roles + - profile + - email + - clientId: svc-polaris-api-admin + name: svc-polaris-api-admin + secret: myPolarisAdminSecret + publicClient: false + # Enable service accounts on the polaris client + serviceAccountsEnabled: true + standardFlowEnabled: false + implicitFlowEnabled: false + directAccessGrantsEnabled: false + defaultClientScopes: + - groups + - roles + - profile + - email + - clientId: svc-spark-etl-polaris-writer + name: svc-spark-etl-polaris-writer + secret: mySparkAppSecret + publicClient: false + # Enable service accounts on the polaris client + serviceAccountsEnabled: true + standardFlowEnabled: false + implicitFlowEnabled: false + directAccessGrantsEnabled: false + defaultClientScopes: + - groups + - roles + - profile + - email + clientScopes: + - name: groups + description: "Groups scope" + protocol: openid-connect + attributes: + include.in.token.scope: "false" + display.on.consent.screen: "true" + protocolMappers: + - name: roles_map_to_groups + protocol: openid-connect + protocolMapper: oidc-usermodel-realm-role-mapper + config: + access.token.claim: "true" + claim.name: groups + id.token.claim: "true" + introspection.token.claim: "true" + jsonType.label: String + lightweight.claim: "true" + multivalued: "true" + userinfo.token.claim: "true" + + # Provision the following databases and configure the provider part. + # -- Sandbox-specific database provisioning for service databases and owners. + defaultDatabaseServer: + # Sandbox implementation-specific + # Consider this part already provided by your platform + # settings: {} + settings: + databases: + - name: hms + owner: + username: "hms" + passwordSecret: creds-hive-metastore-db + - name: superset + owner: + username: "superset" + passwordSecret: creds-superset-db + - name: keycloak + owner: + username: "keycloak" + passwordSecret: creds-keycloak-db + - name: seaweedfs + owner: + username: "seaweedfs" + passwordSecret: creds-seaweedfs-db + - name: polaris + owner: + username: "polaris" + passwordSecret: creds-polaris-db + - name: superset-examples + owner: + username: "examples" + passwordSecret: creds-superset-examples-db + schemas: + - main + - name: airflow + owner: + username: "airflow" + password: "airflow123" + + # Create all the kubernetes secrets required by the services + # The provisioning is local, but you can provide them using external secrets or vault provider + # In order to have a predictable behaviour on re-deployment, we provision the secrets statically + # -- Local static secrets provisioning used by the sandbox examples when no external secret manager is configured. + localSecretsProvider: + info: | + Currently, the secrets are created inside the package `local-secrets-provider.yaml` itself. diff --git a/clusters/sandbox/flux/kubocd.yaml b/clusters/sandbox/flux/kubocd.yaml new file mode 100644 index 0000000..2aa8e47 --- /dev/null +++ b/clusters/sandbox/flux/kubocd.yaml @@ -0,0 +1,84 @@ +# +# Copyright 2026 The OKDP Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +--- +apiVersion: source.toolkit.fluxcd.io/v1 +kind: OCIRepository +metadata: + name: kubocd-controller + namespace: flux-system +spec: + interval: 1m + layerSelector: + mediaType: "application/vnd.cncf.helm.chart.content.v1.tar+gzip" + operation: copy + url: oci://quay.io/kubocd/charts/kubocd-ctrl + ref: + semver: "= v0.2.1" + +--- +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: kubocd-controller + namespace: flux-system +spec: + interval: 1m + serviceAccountName: kustomize-controller + targetNamespace: kubocd + storageNamespace: flux-system + releaseName: kubocd-ctrl + timeout: 10m + install: + createNamespace: true + remediation: + retries: 30 + remediateLastFailure: true + upgrade: + remediation: + retries: 30 + remediateLastFailure: true + chartRef: + kind: OCIRepository + name: kubocd-controller + namespace: flux-system + values: + image: + repository: quay.io/kubocd/kubocd + pullPolicy: Always + extraNamespaces: + - name: kubocd-system + config: + clusterRoles: + - storage + defaultContexts: + - name: platform + namespace: kubocd-system + - name: provider + namespace: kubocd-system + - name: service + namespace: kubocd-system + - name: examples + namespace: kubocd-system + controller: + enabled: true + replicaCount: 1 + logger: + mode: dev + level: info + metrics: + enabled: false + secured: false \ No newline at end of file diff --git a/clusters/sandbox/releases/cert-manager.yaml b/clusters/sandbox/releases/cert-manager.yaml new file mode 100644 index 0000000..2154d78 --- /dev/null +++ b/clusters/sandbox/releases/cert-manager.yaml @@ -0,0 +1,55 @@ +# +# Copyright 2026 The OKDP Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +apiVersion: kubocd.kubotal.io/v1alpha1 +kind: Release +metadata: + name: cert-manager + namespace: kubocd-system +spec: + description: "Certificate management for Kubernetes with trust-manager and issuers" + package: + repository: quay.io/okdp/platform-packages-v0.3/cert-manager + tag: 1.17.1-p06 + interval: 30m + timeout: 10m + parameters: + trust: + enabled: true + bundle: + name: certs-bundle + target: + configMap: + enabled: true + key: "root-certs.pem" + secret: + enabled: true + issuers: + enabled: true + selfSignedClusterIssuers: + - name: default-issuer + certificate: + commonName: "OKDP Sandbox Self-Signed CA" + organization: "OKDP" + country: "FR" + validity: "8760h" + algorithm: "ECDSA" + size: 256 + contexts: + - name: platform + namespace: kubocd-system + targetNamespace: cert-manager + createNamespace: true \ No newline at end of file diff --git a/clusters/sandbox/releases/cloudnative-pg.yaml b/clusters/sandbox/releases/cloudnative-pg.yaml new file mode 100644 index 0000000..9da7e87 --- /dev/null +++ b/clusters/sandbox/releases/cloudnative-pg.yaml @@ -0,0 +1,32 @@ +# +# Copyright 2026 The OKDP Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +apiVersion: kubocd.kubotal.io/v1alpha1 +kind: Release +metadata: + name: cloudnative-pg + namespace: kubocd-system +spec: + createNamespace: true + description: | + CloudNativePG is an open-source Kubernetes operator focused on running and managing PostgreSQL + clusters in Kubernetes environments. + package: + interval: 30m + repository: quay.io/okdp/platform-packages-v0.3/cloudnative-pg + tag: 1.29.1-p01 + timeout: 10m + targetNamespace: cnpg-system \ No newline at end of file diff --git a/clusters/sandbox/releases/cnpg-postgresql.yaml b/clusters/sandbox/releases/cnpg-postgresql.yaml new file mode 100644 index 0000000..feedae5 --- /dev/null +++ b/clusters/sandbox/releases/cnpg-postgresql.yaml @@ -0,0 +1,32 @@ +# +# Copyright 2026 The OKDP Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +apiVersion: kubocd.kubotal.io/v1alpha1 +kind: Release +metadata: + name: cnpg-postgresql + namespace: kubocd-system +spec: + createNamespace: true + description: | + Defines multiple logical PostgreSQL databases managed by the CloudNativePG operator within a single PostgreSQL + instance (cluster) and includes ownership, access credentials, and lifecycle management through Kubernetes-native CRDs. + package: + interval: 30m + repository: quay.io/okdp/platform-packages-v0.3/cnpg-postgresql + tag: 18.3-p01 + timeout: 10m + targetNamespace: cnpg-system \ No newline at end of file diff --git a/clusters/sandbox/releases/coredns-patch.yaml b/clusters/sandbox/releases/coredns-patch.yaml new file mode 100644 index 0000000..f7e2a45 --- /dev/null +++ b/clusters/sandbox/releases/coredns-patch.yaml @@ -0,0 +1,32 @@ +# +# Copyright 2026 The OKDP Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +apiVersion: kubocd.kubotal.io/v1alpha1 +kind: Release +metadata: + name: coredns-patch + namespace: kubocd-system +spec: + description: "CoreDNS configuration patch for local domain resolution" + package: + repository: quay.io/okdp/platform-packages-v0.3/coredns-patch + tag: 1.0.0-p04 + interval: 30m + timeout: 10m + parameters: + ingressService: "ingress-nginx-main-controller.ingress-nginx.svc.cluster.local" + targetNamespace: kube-system + createNamespace: false \ No newline at end of file diff --git a/clusters/sandbox/releases/dns-server.yaml b/clusters/sandbox/releases/dns-server.yaml new file mode 100644 index 0000000..d905c46 --- /dev/null +++ b/clusters/sandbox/releases/dns-server.yaml @@ -0,0 +1,33 @@ +# +# Copyright 2026 The OKDP Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +apiVersion: kubocd.kubotal.io/v1alpha1 +kind: Release +metadata: + name: dns-server + namespace: kubocd-system +spec: + createNamespace: true + description: DNS Server - Lightweight DNS server for local development + package: + interval: 30m + repository: quay.io/okdp/platform-packages-v0.3/dns-server + tag: 1.0.0-p03 + timeout: 10m + targetNamespace: dns-server + parameters: + target: 127.0.0.1 + nodePort: 30053 \ No newline at end of file diff --git a/clusters/sandbox/releases/ingress-nginx.yaml b/clusters/sandbox/releases/ingress-nginx.yaml new file mode 100644 index 0000000..3848777 --- /dev/null +++ b/clusters/sandbox/releases/ingress-nginx.yaml @@ -0,0 +1,34 @@ +# +# Copyright 2026 The OKDP Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +apiVersion: kubocd.kubotal.io/v1alpha1 +kind: Release +metadata: + name: ingress-nginx + namespace: kubocd-system +spec: + description: "NGINX Ingress Controller for HTTP/HTTPS routing" + package: + repository: quay.io/okdp/platform-packages-v0.3/ingress-nginx + tag: 4.12.1-p03 + interval: 30m + timeout: 10m + parameters: + mode: "nodePort" + httpPort: 30080 + httpsPort: 30443 + targetNamespace: ingress-nginx + createNamespace: true \ No newline at end of file diff --git a/clusters/sandbox/releases/keycloak.yaml b/clusters/sandbox/releases/keycloak.yaml new file mode 100644 index 0000000..db0cb9e --- /dev/null +++ b/clusters/sandbox/releases/keycloak.yaml @@ -0,0 +1,35 @@ +# +# Copyright 2026 The OKDP Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +apiVersion: kubocd.kubotal.io/v1alpha1 +kind: Release +metadata: + name: keycloak + namespace: kubocd-system +spec: + createNamespace: true + description: Keycloak identity and access management + package: + interval: 30m + repository: quay.io/okdp/platform-packages-v0.3/keycloak + tag: 24.4.11-p07 + timeout: 10m + parameters: + adminPassword: admin + adminUser: admin + memoryGi: 2 + cpu: 0.5 + targetNamespace: keycloak \ No newline at end of file diff --git a/clusters/sandbox/releases/local-secrets-provider.yaml b/clusters/sandbox/releases/local-secrets-provider.yaml new file mode 100644 index 0000000..aa98cbd --- /dev/null +++ b/clusters/sandbox/releases/local-secrets-provider.yaml @@ -0,0 +1,32 @@ +# +# Copyright 2026 The OKDP Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +apiVersion: kubocd.kubotal.io/v1alpha1 +kind: Release +metadata: + name: local-secrets-provider + namespace: kubocd-system +spec: + createNamespace: true + description: | + A centralized Helm chart for managing and distributing local testing shared configurations, + secrets, services, and environment variables across multiple platform components. + package: + interval: 30m + repository: quay.io/okdp/platform-packages-v0.3/local-secrets-provider + tag: 1.0.0-p03 + timeout: 10m + targetNamespace: default \ No newline at end of file diff --git a/clusters/sandbox/releases/okdp-server.yaml b/clusters/sandbox/releases/okdp-server.yaml new file mode 100644 index 0000000..6ac7933 --- /dev/null +++ b/clusters/sandbox/releases/okdp-server.yaml @@ -0,0 +1,30 @@ +# +# Copyright 2026 The OKDP Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +apiVersion: kubocd.kubotal.io/v1alpha1 +kind: Release +metadata: + name: okdp-server + namespace: kubocd-system +spec: + createNamespace: true + description: OKDP Server - API backend for the Open Kubernetes Data Platform + package: + interval: 30m + repository: quay.io/okdp/platform-packages-v0.3/okdp-server + tag: 0.4.0-p01 + timeout: 10m + targetNamespace: okdp-server \ No newline at end of file diff --git a/clusters/sandbox/releases/okdp-ui.yaml b/clusters/sandbox/releases/okdp-ui.yaml new file mode 100644 index 0000000..6d0ae58 --- /dev/null +++ b/clusters/sandbox/releases/okdp-ui.yaml @@ -0,0 +1,30 @@ +# +# Copyright 2026 The OKDP Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +apiVersion: kubocd.kubotal.io/v1alpha1 +kind: Release +metadata: + name: okdp-ui + namespace: kubocd-system +spec: + createNamespace: true + description: OKDP UI - Frontend web interface for the Open Kubernetes Data Platform + package: + interval: 30m + repository: quay.io/okdp/platform-packages-v0.3/okdp-ui + tag: 0.4.1-p02 + timeout: 10m + targetNamespace: okdp-ui \ No newline at end of file diff --git a/clusters/sandbox/releases/spark-operator.yaml b/clusters/sandbox/releases/spark-operator.yaml new file mode 100644 index 0000000..b19dcda --- /dev/null +++ b/clusters/sandbox/releases/spark-operator.yaml @@ -0,0 +1,17 @@ +apiVersion: kubocd.kubotal.io/v1alpha1 +kind: Release +metadata: + name: spark-operator + namespace: kubocd-system +spec: + description: "Spark Operator for managing Apache Spark applications on Kubernetes" + package: + repository: quay.io/okdp/platform-packages-v0.3/spark-operator + tag: 2.4.0-p02 + parameters: + jobNamespaces: [""] # Allow all namespaces for sandbox simplicity + metricsEnabled: true + podMonitorEnabled: false + webhookEnabled: true + targetNamespace: spark-operator + createNamespace: true diff --git a/clusters/sandbox/releases/spark-rbac.yaml b/clusters/sandbox/releases/spark-rbac.yaml new file mode 100644 index 0000000..e757e6a --- /dev/null +++ b/clusters/sandbox/releases/spark-rbac.yaml @@ -0,0 +1,12 @@ +apiVersion: kubocd.kubotal.io/v1alpha1 +kind: Release +metadata: + name: spark-rbac + namespace: kubocd-system +spec: + description: "Spark ServiceAccount and RBAC for Spark applications" + package: + repository: quay.io/okdp/platform-packages-v0.3/spark-rbac + tag: 1.0.1-p01 + targetNamespace: default + createNamespace: false diff --git a/clusters/sandbox/releases/tools.yaml b/clusters/sandbox/releases/tools.yaml new file mode 100644 index 0000000..a122689 --- /dev/null +++ b/clusters/sandbox/releases/tools.yaml @@ -0,0 +1,38 @@ +# +# Copyright 2026 The OKDP Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +apiVersion: kubocd.kubotal.io/v1alpha1 +kind: Release +metadata: + name: tools + namespace: kubocd-system +spec: + description: "Miscellaneous Kubernetes tools for cluster management" + package: + repository: quay.io/okdp/platform-packages-v0.3/tools + tag: 1.0.0-p01 + interval: 30m + timeout: 10m + parameters: + namespace: kube-tools + reloader: + enabled: true + replicator: + enabled: true + secretGenerator: + enabled: true + targetNamespace: kube-tools + createNamespace: true \ No newline at end of file diff --git a/clusters/sandbox/releases/webhooks.yaml b/clusters/sandbox/releases/webhooks.yaml new file mode 100644 index 0000000..bc44130 --- /dev/null +++ b/clusters/sandbox/releases/webhooks.yaml @@ -0,0 +1,32 @@ +# +# Copyright 2026 The OKDP Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +apiVersion: kubocd.kubotal.io/v1alpha1 +kind: Release +metadata: + name: kubocd-webhooks + namespace: kubocd-system +spec: + description: Second stage of kubocd installation + package: + repository: quay.io/okdp/platform-packages-v0.3/kubocd-webhooks + tag: v0.2.1-p01 + interval: 30m + timeout: 10m + parameters: + replicaCount: 1 + targetNamespace: kubocd + createNamespace: true \ No newline at end of file From 0d056d8e09eb0a9d76332e86846e135f37536657 Mon Sep 17 00:00:00 2001 From: Jean-Pascal MEWENEMESSE Date: Wed, 8 Jul 2026 14:59:24 +0200 Subject: [PATCH 2/4] feat: updated readme to deployment scope only corrected small bug on conventional commit pr trigger make sure to specify flux v2.7.5 because kuboCD does not support latest CRD --- .github/workflows/conventional-commits.yml | 2 +- README.md | 70 +++++++++------------- 2 files changed, 28 insertions(+), 44 deletions(-) diff --git a/.github/workflows/conventional-commits.yml b/.github/workflows/conventional-commits.yml index e2d9e8b..129917c 100644 --- a/.github/workflows/conventional-commits.yml +++ b/.github/workflows/conventional-commits.yml @@ -18,7 +18,7 @@ name: Conventional Commits on: pull_request: - types: [opened, reopened, synchronized] + types: [opened, reopened, synchronize] jobs: commits: diff --git a/README.md b/README.md index c5c19ac..d9d837f 100644 --- a/README.md +++ b/README.md @@ -3,22 +3,10 @@ [![Kubernetes](https://img.shields.io/badge/kubernetes-1.28+-blue.svg)](https://kubernetes.io/) [![Kind](https://img.shields.io/badge/kind-latest-orange.svg)](https://kind.sigs.k8s.io/)   [![License Apache2](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](http://www.apache.org/licenses/LICENSE-2.0) - - - - -![OKDP UI Demo](https://raw.githubusercontent.com/OKDP/okdp-ui/main/docs/images/demo.gif) OKDP Sandbox is a hands-on environment for deploying, testing, and exploring the [OKDP](https://okdp.io) ecosystem on a local Kubernetes cluster. -It provides a ready-to-use data platform environment that includes: -- Identity and access management (Keycloak) -- Object storage (SeaweedFS) -- Spark processing and monitoring (Spark Operator + Spark History Server) -- Workflow orchestration (Apache Airflow) -- Interactive data science workspaces (JupyterHub) -- Data visualization (Apache Superset) -- Platform management (OKDP Server & UI) +It provides a ready-to-use data platform environment covering identity, object storage, Spark processing, workflow orchestration, interactive notebooks, SQL querying, and visualization, running end to end on a local cluster. See [What is included](#what-is-included-in-the-sandbox) for the full component list. ## What is included in the sandbox? @@ -33,19 +21,25 @@ The default OKDP sandbox deploys a complete local data-platform environment with - Apache Superset for dashboards and visual analytics - OKDP Server and OKDP UI for platform management -## Repository status +## Repository scope -This repository is now a lightweight entry point. The reusable deployment assets have been moved to dedicated repositories so they can evolve independently from the sandbox. +This repository owns the **single-cluster sandbox deployment**. It describes how to deploy the OKDP platform onto a local Kubernetes cluster and contains the deployment assets only: -The implementation assets previously owned by this repository were migrated as follows: +- `clusters/sandbox/flux/` : Flux bootstrap of the KuboCD controller (`kubocd.yaml`) +- `clusters/sandbox/releases/` : KuboCD `Release` manifests (what gets installed, which package tag, which parameters) +- `clusters/sandbox/contexts/` : the layered KuboCD `Context` files (`10-platform`, `20-provider`, `30-service`, `99-examples`) +- `docs/` : deployment guides (DNS, certificates) -| Asset | New owner | Reference | -|---|---|---| -| KuboCD packages, release definitions, context layers, package build and release automation | [`OKDP/platform-packages`](https://github.com/OKDP/platform-packages) | [`OKDP/platform-packages#1`](https://github.com/OKDP/platform-packages/pull/1), [`OKDP/okdp-sandbox#51`](https://github.com/OKDP/okdp-sandbox/issues/51) | -| Reusable utility Helm charts | [`OKDP/helm-charts-utilities`](https://github.com/OKDP/helm-charts-utilities) | [`OKDP/helm-charts-utilities#1`](https://github.com/OKDP/helm-charts-utilities/pull/1), [`OKDP/okdp-sandbox#52`](https://github.com/OKDP/okdp-sandbox/issues/52) | -| Notebooks, DAGs, and runnable examples | [`OKDP/okdp-examples`](https://github.com/OKDP/okdp-examples) | Examples repository | +The **packages themselves** (the KuboCD packages bundled as OCI artifacts) live in a dedicated repository and are consumed here from the registry. This repository never builds packages, it only deploys published ones. -As a result, this repository should no longer contain duplicated `.github/`, `clusters/`, or `packages/` directories. +| Concern | Owner | +|---|---| +| KuboCD packages (system and services), build and release automation | [`OKDP/platform-packages`](https://github.com/OKDP/platform-packages) | +| Reusable utility Helm charts | [`OKDP/helm-charts-utilities`](https://github.com/OKDP/helm-charts-utilities) | +| Notebooks, DAGs, and runnable examples | [`OKDP/okdp-examples`](https://github.com/OKDP/okdp-examples) | +| OKDP web UI | [`OKDP/okdp-ui`](https://github.com/OKDP/okdp-ui) | +| OKDP backend server (API) | [`OKDP/okdp-server`](https://github.com/OKDP/okdp-server) | +| Single-cluster sandbox deployment (this repository) | `OKDP/okdp-sandbox` | ## Prerequisites @@ -59,17 +53,17 @@ As a result, this repository should no longer contain duplicated `.github/`, `cl - [Docker](https://docs.docker.com/get-docker/) or a compatible container runtime - [Kind](https://kind.sigs.k8s.io/docs/user/quick-start/#installation) - [kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/) -- [Flux CLI](https://fluxcd.io/flux/installation/) +- [Flux CLI v2.7.5](https://fluxcd.io/flux/installation/) ## Quick start -The maintained sandbox deployment files now live in [`OKDP/platform-packages`](https://github.com/OKDP/platform-packages). +The sandbox deployment files live in this repository under `clusters/sandbox/`. -### 1. Clone the platform packages repository +### 1. Clone this repository ```sh -git clone https://github.com/OKDP/platform-packages.git -cd platform-packages +git clone https://github.com/OKDP/okdp-sandbox.git +cd okdp-sandbox ``` ### 2. Create Kubernetes Kind Cluster @@ -213,7 +207,7 @@ kubectl wait --for=condition=ready pod -l app=source-controller -n flux-system - > It manages platform components and applications **declaratively**, providing a higher-level CD abstraction for GitOps workflows. ```sh -kubectl apply -f kubocd.yaml +kubectl apply -f clusters/sandbox/flux/kubocd.yaml ``` #### Deploy/Upgrade OKDP platform components @@ -262,10 +256,10 @@ Deploy/Upgrade the sandbox default context: ```sh -kubectl apply -f 10-platform-context.yaml -kubectl apply -f 20-provider-context.yaml -kubectl apply -f 30-service-context.yaml -kubectl apply -f 99-examples-context.yaml +kubectl apply -f clusters/sandbox/contexts/10-platform-context.yaml +kubectl apply -f clusters/sandbox/contexts/20-provider-context.yaml +kubectl apply -f clusters/sandbox/contexts/30-service-context.yaml +kubectl apply -f clusters/sandbox/contexts/99-examples-context.yaml ``` > 💡 By default, the **default Context** uses **okdp.sandbox** as the ingress domain suffix. @@ -316,7 +310,7 @@ spec: Deploy/Upgrade OKDP components: ```sh -kubectl apply -f releases/ +kubectl apply -f clusters/sandbox/releases/ ``` #### Verify and monitor release deployment status @@ -392,16 +386,6 @@ Remove-Item "$env:TEMP\okdp-sandbox-config.yaml" -Force -## Related repositories - -| Repository | Purpose | -|---|---| -| [`OKDP/platform-packages`](https://github.com/OKDP/platform-packages) | KuboCD packages, release definitions, context layers, and deployment assets | -| [`OKDP/helm-charts-utilities`](https://github.com/OKDP/helm-charts-utilities) | Reusable utility Helm charts consumed by packages and examples | -| [`OKDP/okdp-examples`](https://github.com/OKDP/okdp-examples) | Notebooks, DAGs, and data-platform examples | -| [`OKDP/okdp-ui`](https://github.com/OKDP/okdp-ui) | OKDP web UI | -| [`OKDP/okdp-server`](https://github.com/OKDP/okdp-server) | OKDP backend server | - ## License This project is licensed under the [Apache License 2.0](http://www.apache.org/licenses/LICENSE-2.0). From 160b9644ebfb6dc76c6f4c1875de6044db5dd694 Mon Sep 17 00:00:00 2001 From: Jean-Pascal MEWENEMESSE Date: Wed, 8 Jul 2026 15:24:46 +0200 Subject: [PATCH 3/4] ci: add end-to*end sandbox deploy validation Add a Kind + FLux + KuboCD workflow that applies clusters/sandbox/{contexts,releases}/ keep package build publish stuffs in platform-packages --- .github/workflows/deploy-validation.yml | 199 ++++++++++++++++++ clusters/sandbox/flux/kubocd.yaml | 2 +- clusters/sandbox/releases/cert-manager.yaml | 2 +- clusters/sandbox/releases/cloudnative-pg.yaml | 2 +- .../sandbox/releases/cnpg-postgresql.yaml | 2 +- clusters/sandbox/releases/coredns-patch.yaml | 2 +- clusters/sandbox/releases/dns-server.yaml | 2 +- clusters/sandbox/releases/ingress-nginx.yaml | 2 +- clusters/sandbox/releases/keycloak.yaml | 2 +- .../releases/local-secrets-provider.yaml | 2 +- clusters/sandbox/releases/okdp-server.yaml | 2 +- clusters/sandbox/releases/okdp-ui.yaml | 2 +- clusters/sandbox/releases/tools.yaml | 2 +- clusters/sandbox/releases/webhooks.yaml | 2 +- 14 files changed, 212 insertions(+), 13 deletions(-) create mode 100644 .github/workflows/deploy-validation.yml diff --git a/.github/workflows/deploy-validation.yml b/.github/workflows/deploy-validation.yml new file mode 100644 index 0000000..4d98e7c --- /dev/null +++ b/.github/workflows/deploy-validation.yml @@ -0,0 +1,199 @@ +# +# Copyright 2026 The OKDP Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# End-to-end install validation for the sandbox deployment. +# +# Spins up a throwaway Kind cluster, installs Flux and the KuboCD controller, +# then applies the contexts and releases from clusters/sandbox/ exactly as the +# README instructs and waits until every KuboCD Release reaches READY. +# +# Unlike the package CI in OKDP/platform-packages, this workflow builds nothing: +# it deploys the already-published packages (quay.io/okdp/platform-packages/) +# straight from the manifests, so it validates the deployment layer, not packages. +# +# NOTE: the releases reference quay.io/okdp/platform-packages/ (no version +# suffix), which only exists once OKDP/platform-packages#19 has merged and +# republished. Until then this workflow is expected to fail. + +name: Deploy validation (sandbox e2e) + +on: + pull_request: + paths: + - "clusters/**" + - ".github/workflows/deploy-validation.yml" + push: + branches: [main] + paths: + - "clusters/**" + workflow_dispatch: + +defaults: + run: + shell: bash + +jobs: + deploy-validation: + # A larger runner gives the full platform room to schedule; on a small runner + # the capacity escape-hatch below turns "insufficient cpu/memory" into a pass. + runs-on: ubuntu-latest + steps: + + - name: Checkout repo ⚡️ + uses: actions/checkout@v4 + + - name: Free up disk space 📦 + uses: okdp/gh-workflows/.github/actions/free-disk-space@v1 + + - name: Set up Kind cluster 📦 + uses: okdp/gh-workflows/.github/actions/setup-kind@v1 + + - name: Setup Flux CLI 🛠️ + uses: fluxcd/flux2/action@main + with: + version: "2.6.4" + + - name: Deploy Flux (basic mode) 🚀 + run: flux install + + - name: Wait until Flux controllers are ready ⌛ + run: | + kubectl wait --for=condition=ready pod -l app=source-controller -n flux-system --timeout=300s + + - name: Verify Flux installation ✅ + run: | + flux version + kubectl -n flux-system get pods + + - name: Bootstrap the KuboCD controller 🛠️ + run: kubectl apply -f clusters/sandbox/flux/kubocd.yaml + + - name: Wait until the KuboCD controller is ready ⌛ + run: | + # HelmRelease "Ready" implies the controller is rolled out and its CRDs + # (Context, Release) are registered, which the next steps need. + kubectl wait --for=condition=Ready helmrelease/kubocd-controller \ + -n flux-system --timeout=600s + kubectl -n kubocd get pods + + - name: Apply the platform contexts 🚀 + run: | + # Order is immaterial: precedence is resolved by defaultContexts in + # kubocd.yaml. The NN- filename prefixes only order this glob apply. + kubectl apply -f clusters/sandbox/contexts/ + + - name: Apply the platform releases 🚀 + run: | + kubectl apply -f clusters/sandbox/releases/ + + - name: Wait for all releases to become READY ✅ + run: | + echo "**************************************** Storage Classes ****************************************" + kubectl get sc + kubectl get events -A -w >/tmp/kubectl-events.log 2>&1 & + EVENTS_PID=$! + trap 'kill $EVENTS_PID 2>/dev/null || true' EXIT + + # Poll release phase explicitly; more stable than kubectl wait for this CRD. + timeout_secs=1200 + interval_secs=15 + elapsed=0 + ready=0 + + while [ "$elapsed" -lt "$timeout_secs" ]; do + echo "Wait until all KuboCD releases are ready ..." + echo "************************************** Releases status **************************************" + kubectl get release -A || true + echo "**************************************** Pods status ****************************************" + kubectl get po -A || true + echo "**************************************** PVC status ****************************************" + kubectl get pvc -A || true + echo "**************************************** Pending Pods ****************************************" + kubectl get pods -A --field-selector=status.phase=Pending \ + -o custom-columns=NS:.metadata.namespace,NAME:.metadata.name --no-headers \ + | xargs -r -n2 sh -c 'echo "Pending Pod: $0/$1 ==="; kubectl describe pod -n "$0" "$1"; echo' + + echo "**************************************** Unhealthy Pods ****************************************" + kubectl get pods -A --no-headers \ + | awk '$4 ~ /CrashLoopBackOff|Error|ImagePullBackOff|CreateContainerConfigError|CreateContainerError/ {print $1, $2}' \ + | while read -r ns pod; do + [ -n "$ns" ] || continue + [ -n "$pod" ] || continue + echo "Unhealthy Pod: $ns/$pod ===" + kubectl describe pod -n "$ns" "$pod" || true + containers="$( + kubectl get pod -n "$ns" "$pod" -o jsonpath='{range .spec.initContainers[*]}{.name}{"\n"}{end}{range .spec.containers[*]}{.name}{"\n"}{end}' 2>/dev/null || true + )" + for c in $containers; do + echo "--- logs current: $ns/$pod container=$c ---" + kubectl logs -n "$ns" "$pod" -c "$c" --tail=200 || true + echo "--- logs previous: $ns/$pod container=$c ---" + kubectl logs -n "$ns" "$pod" -c "$c" --previous --tail=200 || true + done + echo + done + + not_ready="$(kubectl get release -A -o jsonpath='{range .items[?(@.status.phase!="READY")]}{.metadata.namespace}/{.metadata.name}{"\n"}{end}' 2>/dev/null || true)" + + if [ -z "$not_ready" ]; then + echo "All releases are READY." + ready=1 + break + fi + + echo "Releases not ready yet:" + echo "$not_ready" + + scheduling_errors="$( + kubectl get events -A --field-selector reason=FailedScheduling --no-headers 2>/dev/null \ + | grep -E 'Insufficient (cpu|memory)' || true + )" + + if [ -n "$scheduling_errors" ]; then + echo "Detected insufficient CPU or memory in Kubernetes scheduling events." + echo "Treating this as a CI capacity limitation and exiting successfully." + echo "$scheduling_errors" + exit 0 + fi + + sleep "$interval_secs" + elapsed=$((elapsed + interval_secs)) + done + + if [ "$ready" -eq 1 ]; then + exit 0 + fi + + echo "Timed out waiting for READY releases after ${timeout_secs}s" + kubectl get release -A + kubectl get helmrelease -A || true + kubectl get jobs -A || true + echo "Last events log lines:" + tail -n 200 /tmp/kubectl-events.log || true + + scheduling_errors="$( + kubectl get events -A --field-selector reason=FailedScheduling --no-headers 2>/dev/null \ + | grep -E 'Insufficient (cpu|memory)' || true + )" + + if [ -n "$scheduling_errors" ]; then + echo "Detected insufficient CPU or memory in Kubernetes scheduling events." + echo "Treating this as a CI capacity limitation and exiting successfully." + echo "$scheduling_errors" + exit 0 + fi + + exit 1 diff --git a/clusters/sandbox/flux/kubocd.yaml b/clusters/sandbox/flux/kubocd.yaml index 2aa8e47..221eaf2 100644 --- a/clusters/sandbox/flux/kubocd.yaml +++ b/clusters/sandbox/flux/kubocd.yaml @@ -81,4 +81,4 @@ spec: level: info metrics: enabled: false - secured: false \ No newline at end of file + secured: false diff --git a/clusters/sandbox/releases/cert-manager.yaml b/clusters/sandbox/releases/cert-manager.yaml index 2154d78..91827de 100644 --- a/clusters/sandbox/releases/cert-manager.yaml +++ b/clusters/sandbox/releases/cert-manager.yaml @@ -52,4 +52,4 @@ spec: - name: platform namespace: kubocd-system targetNamespace: cert-manager - createNamespace: true \ No newline at end of file + createNamespace: true diff --git a/clusters/sandbox/releases/cloudnative-pg.yaml b/clusters/sandbox/releases/cloudnative-pg.yaml index 9da7e87..bb0e826 100644 --- a/clusters/sandbox/releases/cloudnative-pg.yaml +++ b/clusters/sandbox/releases/cloudnative-pg.yaml @@ -29,4 +29,4 @@ spec: repository: quay.io/okdp/platform-packages-v0.3/cloudnative-pg tag: 1.29.1-p01 timeout: 10m - targetNamespace: cnpg-system \ No newline at end of file + targetNamespace: cnpg-system diff --git a/clusters/sandbox/releases/cnpg-postgresql.yaml b/clusters/sandbox/releases/cnpg-postgresql.yaml index feedae5..a7e8fb8 100644 --- a/clusters/sandbox/releases/cnpg-postgresql.yaml +++ b/clusters/sandbox/releases/cnpg-postgresql.yaml @@ -29,4 +29,4 @@ spec: repository: quay.io/okdp/platform-packages-v0.3/cnpg-postgresql tag: 18.3-p01 timeout: 10m - targetNamespace: cnpg-system \ No newline at end of file + targetNamespace: cnpg-system diff --git a/clusters/sandbox/releases/coredns-patch.yaml b/clusters/sandbox/releases/coredns-patch.yaml index f7e2a45..9d6ca2e 100644 --- a/clusters/sandbox/releases/coredns-patch.yaml +++ b/clusters/sandbox/releases/coredns-patch.yaml @@ -29,4 +29,4 @@ spec: parameters: ingressService: "ingress-nginx-main-controller.ingress-nginx.svc.cluster.local" targetNamespace: kube-system - createNamespace: false \ No newline at end of file + createNamespace: false diff --git a/clusters/sandbox/releases/dns-server.yaml b/clusters/sandbox/releases/dns-server.yaml index d905c46..b532e88 100644 --- a/clusters/sandbox/releases/dns-server.yaml +++ b/clusters/sandbox/releases/dns-server.yaml @@ -30,4 +30,4 @@ spec: targetNamespace: dns-server parameters: target: 127.0.0.1 - nodePort: 30053 \ No newline at end of file + nodePort: 30053 diff --git a/clusters/sandbox/releases/ingress-nginx.yaml b/clusters/sandbox/releases/ingress-nginx.yaml index 3848777..9c9fee3 100644 --- a/clusters/sandbox/releases/ingress-nginx.yaml +++ b/clusters/sandbox/releases/ingress-nginx.yaml @@ -31,4 +31,4 @@ spec: httpPort: 30080 httpsPort: 30443 targetNamespace: ingress-nginx - createNamespace: true \ No newline at end of file + createNamespace: true diff --git a/clusters/sandbox/releases/keycloak.yaml b/clusters/sandbox/releases/keycloak.yaml index db0cb9e..ae4cafd 100644 --- a/clusters/sandbox/releases/keycloak.yaml +++ b/clusters/sandbox/releases/keycloak.yaml @@ -32,4 +32,4 @@ spec: adminUser: admin memoryGi: 2 cpu: 0.5 - targetNamespace: keycloak \ No newline at end of file + targetNamespace: keycloak diff --git a/clusters/sandbox/releases/local-secrets-provider.yaml b/clusters/sandbox/releases/local-secrets-provider.yaml index aa98cbd..5039ff7 100644 --- a/clusters/sandbox/releases/local-secrets-provider.yaml +++ b/clusters/sandbox/releases/local-secrets-provider.yaml @@ -29,4 +29,4 @@ spec: repository: quay.io/okdp/platform-packages-v0.3/local-secrets-provider tag: 1.0.0-p03 timeout: 10m - targetNamespace: default \ No newline at end of file + targetNamespace: default diff --git a/clusters/sandbox/releases/okdp-server.yaml b/clusters/sandbox/releases/okdp-server.yaml index 6ac7933..65e21b7 100644 --- a/clusters/sandbox/releases/okdp-server.yaml +++ b/clusters/sandbox/releases/okdp-server.yaml @@ -27,4 +27,4 @@ spec: repository: quay.io/okdp/platform-packages-v0.3/okdp-server tag: 0.4.0-p01 timeout: 10m - targetNamespace: okdp-server \ No newline at end of file + targetNamespace: okdp-server diff --git a/clusters/sandbox/releases/okdp-ui.yaml b/clusters/sandbox/releases/okdp-ui.yaml index 6d0ae58..32e385e 100644 --- a/clusters/sandbox/releases/okdp-ui.yaml +++ b/clusters/sandbox/releases/okdp-ui.yaml @@ -27,4 +27,4 @@ spec: repository: quay.io/okdp/platform-packages-v0.3/okdp-ui tag: 0.4.1-p02 timeout: 10m - targetNamespace: okdp-ui \ No newline at end of file + targetNamespace: okdp-ui diff --git a/clusters/sandbox/releases/tools.yaml b/clusters/sandbox/releases/tools.yaml index a122689..e80e2dc 100644 --- a/clusters/sandbox/releases/tools.yaml +++ b/clusters/sandbox/releases/tools.yaml @@ -35,4 +35,4 @@ spec: secretGenerator: enabled: true targetNamespace: kube-tools - createNamespace: true \ No newline at end of file + createNamespace: true diff --git a/clusters/sandbox/releases/webhooks.yaml b/clusters/sandbox/releases/webhooks.yaml index bc44130..b607d39 100644 --- a/clusters/sandbox/releases/webhooks.yaml +++ b/clusters/sandbox/releases/webhooks.yaml @@ -29,4 +29,4 @@ spec: parameters: replicaCount: 1 targetNamespace: kubocd - createNamespace: true \ No newline at end of file + createNamespace: true From 1a724abfaa04baee6ea33664c5d76d48daf8a3c4 Mon Sep 17 00:00:00 2001 From: Jean-Pascal MEWENEMESSE Date: Wed, 8 Jul 2026 15:44:48 +0200 Subject: [PATCH 4/4] fix: updated the correct flux version to match with the readme pined version --- .github/workflows/deploy-validation.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy-validation.yml b/.github/workflows/deploy-validation.yml index 4d98e7c..d23a1fb 100644 --- a/.github/workflows/deploy-validation.yml +++ b/.github/workflows/deploy-validation.yml @@ -64,7 +64,7 @@ jobs: - name: Setup Flux CLI 🛠️ uses: fluxcd/flux2/action@main with: - version: "2.6.4" + version: "2.7.5" - name: Deploy Flux (basic mode) 🚀 run: flux install