From 8fbbcb75fd69bfe094e882f43a3cf1787dd00a51 Mon Sep 17 00:00:00 2001 From: iizitounene Date: Thu, 19 Mar 2026 15:49:54 +0100 Subject: [PATCH 1/3] docs: add helm docs + licence headers --- helm/okdp-examples/Chart.yaml | 4 +- helm/okdp-examples/templates/_helpers.tpl | 18 + helm/okdp-examples/templates/configmap.yaml | 32 +- helm/okdp-examples/templates/job.yaml | 34 +- .../templates/serviceaccount.yaml | 32 +- helm/okdp-examples/values.yaml | 402 +++++++++--------- 6 files changed, 282 insertions(+), 240 deletions(-) diff --git a/helm/okdp-examples/Chart.yaml b/helm/okdp-examples/Chart.yaml index a2ff3a1..cc331c1 100644 --- a/helm/okdp-examples/Chart.yaml +++ b/helm/okdp-examples/Chart.yaml @@ -17,10 +17,10 @@ apiVersion: v2 type: application name: okdp-examples -version: 1.1.0 +version: 1.1.1 sources: - https://github.com/okdp/okdp-examples -appVersion: "1.1.0" +appVersion: "1.1.1" description: | A collection of hands-on examples, helper utilities, Jupyter notebooks, and data workflows that showcases how to work with the OKDP Platform. diff --git a/helm/okdp-examples/templates/_helpers.tpl b/helm/okdp-examples/templates/_helpers.tpl index 23f9f2e..d735331 100644 --- a/helm/okdp-examples/templates/_helpers.tpl +++ b/helm/okdp-examples/templates/_helpers.tpl @@ -1,3 +1,21 @@ +{{/* + + Copyright 2026 The OKDP Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +*/}} + {{/* Expand the name of the chart. */}} diff --git a/helm/okdp-examples/templates/configmap.yaml b/helm/okdp-examples/templates/configmap.yaml index 6cbddc9..4566eee 100644 --- a/helm/okdp-examples/templates/configmap.yaml +++ b/helm/okdp-examples/templates/configmap.yaml @@ -1,18 +1,20 @@ -# -# Copyright 2026 The OKDP Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# +{{/* + + Copyright 2026 The OKDP Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +*/}} {{- $root := . }} {{- $groupNames := sortAlpha (keys $root.Values.commands) }} diff --git a/helm/okdp-examples/templates/job.yaml b/helm/okdp-examples/templates/job.yaml index 1df9f2c..34a736f 100644 --- a/helm/okdp-examples/templates/job.yaml +++ b/helm/okdp-examples/templates/job.yaml @@ -1,21 +1,23 @@ -# -# Copyright 2026 The OKDP Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# +{{/* + + Copyright 2026 The OKDP Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +*/}} {{- $root := . }} - {{- $groupNames := sortAlpha (keys $root.Values.commands) }} +{{- $groupNames := sortAlpha (keys $root.Values.commands) }} apiVersion: batch/v1 kind: Job metadata: diff --git a/helm/okdp-examples/templates/serviceaccount.yaml b/helm/okdp-examples/templates/serviceaccount.yaml index 655aa9d..8a3732c 100644 --- a/helm/okdp-examples/templates/serviceaccount.yaml +++ b/helm/okdp-examples/templates/serviceaccount.yaml @@ -1,18 +1,20 @@ -# -# Copyright 2026 The OKDP Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# +{{/* + + Copyright 2026 The OKDP Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +*/}} {{- if .Values.serviceAccount.create -}} apiVersion: v1 diff --git a/helm/okdp-examples/values.yaml b/helm/okdp-examples/values.yaml index 91ecab0..9550ce6 100644 --- a/helm/okdp-examples/values.yaml +++ b/helm/okdp-examples/values.yaml @@ -14,28 +14,36 @@ # limitations under the License. # -# Default values for okdp-examples. -# This is a YAML-formatted file. -# Declare variables to be passed into your templates. - +# -- Container image configuration for the examples Job. image: + # -- OCI image repository for the examples container. repository: quay.io/okdp/okdp-examples + # -- Kubernetes image pull policy. pullPolicy: Always - # Overrides the image tag whose default is the chart appVersion. + # -- Image tag to use. Defaults are not inferred here; this value is explicit. tag: "latest" +# -- Image pull secrets for private registries. imagePullSecrets: [] + +# -- Override the chart name. nameOverride: "" + +# -- Override the fully qualified release name. fullnameOverride: "" +# -- Kubernetes Job configuration. job: + # -- Pod restart policy for the Job. restartPolicy: Never + # -- Number of retries before the Job is considered failed. backoffLimit: 2 + # -- Annotations applied to the Job metadata, typically used for Helm hooks. annotations: "helm.sh/hook": post-install,post-upgrade "helm.sh/hook-delete-policy": "before-hook-creation" -# -- Extra environment variables in RAW format that will be passed into pods +# -- Extra environment variables in raw Kubernetes env format injected into the Job container. extraEnvRaw: [] # - name: MC_INSECURE # value: "1" @@ -64,206 +72,213 @@ extraEnvRaw: [] # - name: SPARK_EVENT_LOG_DIRECTORY # value: s3a://spark-events/event-logs/ -commands: - 01-prepare_env: - 01-create-spark-event-log-dir: | - echo "👉 Connect into S3" - mc alias set myminio $S3_ENDPOINT "$S3_ACCESS_KEY" "$S3_SECRET_KEY" - echo "👉 MinIO alias configured successfully." - echo "👉 Normalize the log directory $SPARK_EVENT_LOG_DIRECTORY" - no_scheme="${SPARK_EVENT_LOG_DIRECTORY#s3a://}" - no_scheme="${no_scheme%/}" - bucket="${no_scheme%%/*}" - prefix="${no_scheme#${bucket}}" - prefix="${prefix#/}" - echo "👉 Bucket: $bucket, Prefix: $prefix" - echo "👉 Create the bucket $bucket if not exist" - mc mb myminio/$bucket || true - echo "👉 upload a zero-byte object to create the prefix: ${bucket}/${prefix}/.keep" - mc pipe "myminio/${bucket}/${prefix}/.keep" < /dev/null - echo "✅ Spark events log directory $SPARK_EVENT_LOG_DIRECTORY successfully created."; - 02-nyc_trip: - # https://www.nyc.gov/site/tlc/about/tlc-trip-record-data.page - 01-download-nyc-tripdata: | - mkdir -p /data/tripdata - cd /data/tripdata; - for dataset in yellow green fhv; do - for month in 01 02 03; do - file="${dataset}_tripdata_2025-${month}.parquet"; - url="$DATA_URL/$file"; - echo "→ $url"; - if curl -fsSLO "$url"; then - echo "✅ Downloaded: $file"; - else - echo "⚠️ Missing: $file"; - fi; - done; - done; - ls -lh /data/tripdata - echo "✅ All downloads complete."; - - 02-upload-nyc-tripdata-into-s3: | - echo "👉 Connect into S3" - mc alias set myminio $S3_ENDPOINT "$S3_ACCESS_KEY" "$S3_SECRET_KEY" - echo "👉 MinIO alias configured successfully." - ### echo "👉 Clean the examples folder if it exists" - ### mc rm --recursive --force myminio/$BUCKET/examples - echo "👉 Create the bucket if not exist" - mc mb myminio/$BUCKET || true - echo "👉 Upload sample files into the correct folders" - - for category in yellow green fhv; do - echo "🚀 Processing category: $category" - for file in /data/tripdata/${category}_*.parquet; do - [ -e "$file" ] || continue # skip if no files match - month=$(basename "$file" | sed -E 's/.*_([0-9]{4}-[0-9]{2})\.parquet/\1/') - echo "📦 Uploading $file → ${BUCKET_PREFIX}/$category/month=$month/" - mc cp "$file" "myminio/$BUCKET/${BUCKET_PREFIX}/$category/month=$month/" - done - done - - echo "👉 Verify bucket structure" - mc tree myminio/$BUCKET - mc ls --recursive myminio/$BUCKET - - 03-nyc-tripdata-trino-external-tables.sql: | - trino --server ${TRINO_SERVER_URL} --insecure < 'nyc_tripdata', - table_name => 'yellow', - mode => 'ADD' - ); - CALL lakehouse.system.sync_partition_metadata( - schema_name => 'nyc_tripdata', - table_name => 'green', - mode => 'ADD' - ); - CALL lakehouse.system.sync_partition_metadata( - schema_name => 'nyc_tripdata', - table_name => 'fhv', - mode => 'ADD' - ); - SQL - - 05-nyc-tripdata-trino-validate: | - trino --server ${TRINO_SERVER_URL} --insecure < 'nyc_tripdata', +# table_name => 'yellow', +# mode => 'ADD' +# ); +# CALL lakehouse.system.sync_partition_metadata( +# schema_name => 'nyc_tripdata', +# table_name => 'green', +# mode => 'ADD' +# ); +# CALL lakehouse.system.sync_partition_metadata( +# schema_name => 'nyc_tripdata', +# table_name => 'fhv', +# mode => 'ADD' +# ); +# SQL +# +# 05-nyc-tripdata-trino-validate: | +# trino --server ${TRINO_SERVER_URL} --insecure < Date: Fri, 22 May 2026 19:56:41 +0200 Subject: [PATCH 2/3] chore: release 1.1.0 Release-As: 1.1.0 --- docker/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/package.json b/docker/package.json index d068540..0876a71 100644 --- a/docker/package.json +++ b/docker/package.json @@ -1,6 +1,6 @@ { "name": "okdp-examples", - "version": "1.0.0", + "version": "1.1.0", "description": "okdp-examples docker image", "repository": { "type": "git", From 49b47e85bf9fadfe16744aa45bb6aa13fbd369af Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 22 May 2026 17:57:20 +0000 Subject: [PATCH 3/3] [helm-docs] Update readme --- helm/okdp-examples/README.md | 61 +++++++++++++++++------------------- 1 file changed, 29 insertions(+), 32 deletions(-) diff --git a/helm/okdp-examples/README.md b/helm/okdp-examples/README.md index 08dcfb7..19ab3fb 100644 --- a/helm/okdp-examples/README.md +++ b/helm/okdp-examples/README.md @@ -1,6 +1,6 @@ # okdp-examples -![Version: 1.1.0](https://img.shields.io/badge/Version-1.1.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 1.1.0](https://img.shields.io/badge/AppVersion-1.1.0-informational?style=flat-square) +![Version: 1.1.1](https://img.shields.io/badge/Version-1.1.1-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 1.1.1](https://img.shields.io/badge/AppVersion-1.1.1-informational?style=flat-square) A collection of hands-on examples, helper utilities, Jupyter notebooks, and data workflows that showcases how to work with the OKDP Platform. @@ -19,37 +19,34 @@ that showcases how to work with the OKDP Platform. | Key | Type | Default | Description | |-----|------|---------|-------------| -| affinity | object | `{}` | | -| commands.01-prepare_env.01-create-spark-event-log-dir | string | `"echo \"👉 Connect into S3\"\nmc alias set myminio $S3_ENDPOINT \"$S3_ACCESS_KEY\" \"$S3_SECRET_KEY\"\necho \"👉 MinIO alias configured successfully.\"\necho \"👉 Normalize the log directory $SPARK_EVENT_LOG_DIRECTORY\"\nno_scheme=\"${SPARK_EVENT_LOG_DIRECTORY#s3a://}\"\nno_scheme=\"${no_scheme%/}\"\nbucket=\"${no_scheme%%/*}\"\nprefix=\"${no_scheme#${bucket}}\"\nprefix=\"${prefix#/}\"\necho \"👉 Bucket: $bucket, Prefix: $prefix\"\necho \"👉 Create the bucket $bucket if not exist\"\nmc mb myminio/$bucket || true\necho \"👉 upload a zero-byte object to create the prefix: ${bucket}/${prefix}/.keep\"\nmc pipe \"myminio/${bucket}/${prefix}/.keep\" < /dev/null\necho \"✅ Spark events log directory $SPARK_EVENT_LOG_DIRECTORY successfully created.\";\n"` | | -| commands.02-nyc_trip."03-nyc-tripdata-trino-external-tables.sql" | string | `"trino --server ${TRINO_SERVER_URL} --insecure < 'nyc_tripdata',\n table_name => 'yellow',\n mode => 'ADD'\n );\n CALL lakehouse.system.sync_partition_metadata(\n schema_name => 'nyc_tripdata',\n table_name => 'green',\n mode => 'ADD'\n );\n CALL lakehouse.system.sync_partition_metadata(\n schema_name => 'nyc_tripdata',\n table_name => 'fhv',\n mode => 'ADD'\n );\nSQL\n"` | | -| commands.02-nyc_trip.05-nyc-tripdata-trino-validate | string | `"trino --server ${TRINO_SERVER_URL} --insecure <