diff --git a/.dockerignore b/.dockerignore index b20cfb8..a5d8001 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,3 +1,6 @@ jupyterhub values.yaml config.yml +config-generator +skaffold.yaml +files/* \ No newline at end of file diff --git a/.github/workflows/.github-ci.yaml b/.github/workflows/.github-ci.yaml new file mode 100644 index 0000000..81eb52a --- /dev/null +++ b/.github/workflows/.github-ci.yaml @@ -0,0 +1,171 @@ +name: Build, Test, and Deploy Docker Image + +on: + push: + branches: + - develop + +jobs: + build: + runs-on: ubuntu-latest + steps: + # Step 1: Checkout repository + - uses: actions/checkout@v4 + + # Step 2: Install Trivy + - name: Install Trivy + run: | + sudo apt-get update -y + sudo apt-get install -y wget apt-transport-https gnupg lsb-release + wget -qO - https://aquasecurity.github.io/trivy-repo/deb/public.key | sudo apt-key add - + echo deb https://aquasecurity.github.io/trivy-repo/deb $(lsb_release -sc) main | sudo tee -a /etc/apt/sources.list.d/trivy.list + sudo apt-get update -y + sudo apt-get install -y trivy + + # Step 3: Read image name + - name: Read docker name + id: yaml-docker-name + uses: jbutcher5/read-yaml@main + with: + file: 'build.yml' + key-path: '["docker_image_name"]' + + # Step 4: Read image version + - name: Read docker version + id: yaml-docker-version + uses: jbutcher5/read-yaml@main + with: + file: 'build.yml' + key-path: '["docker_image_version"]' + + # Step 5: Generate Docker tag + - name: Generate docker tag + env: + GITHUB_BRANCH: ${{ github.ref }} + docker_image_name: ${{ steps.yaml-docker-name.outputs.data }} + docker_image_version: ${{ steps.yaml-docker-version.outputs.data }} + run: | + branch_name=${GITHUB_BRANCH#refs/heads/} + echo "branch_name=${GITHUB_BRANCH#refs/heads/}" >> $GITHUB_ENV + if [[ "$branch_name" = "main" ]] + then + mType="" + else + mType="dev" + fi + echo "docker_tag=$docker_image_name:$docker_image_version" >> $GITHUB_ENV + echo "docker_tag_latest=$docker_image_name:latest" >> $GITHUB_ENV + docker_image_application=(${docker_image_name#*/}) + echo "docker_image_application=$docker_image_application" >> $GITHUB_ENV + echo "docker_image_version=$docker_image_version" >> $GITHUB_ENV + + # Step 6: Build Docker image to inspect it with Trivy + - name: Build Docker image + run: | + tag="${docker_image_application}:${docker_image_version}" + echo "${{ secrets.CR_PASSWORD }}" | docker login -u "${{ secrets.CR_USERNAME }}" --password-stdin "${{ secrets.CR_REGISTRY }}" + docker build -t "${tag}" --file Dockerfile . + + # Step 7: Save Docker image as tar.gz + - name: Save Docker Image as tar.gz + run: | + tag="${docker_image_application}:${docker_image_version}" + docker save "${tag}" -o "${docker_image_application}_${docker_image_version}.tar" + tar -czf "${docker_image_application}_${docker_image_version}.tar.gz" "${docker_image_application}_${docker_image_version}.tar" + + # Step 8: Upload Docker Image tar.gz as an artifact + - name: Upload Docker Image Artifact + uses: actions/upload-artifact@v3 + with: + name: docker-image-tar + path: ${{ env.docker_image_application }}_${{ env.docker_image_version }}.tar.gz + + + # Step 9: Scan Docker Image with Trivy + - name: Scan Docker Image with Trivy + run: | + tag="${docker_image_application}:${docker_image_version}" + trivy image --no-progress --exit-code 1 --severity HIGH,CRITICAL,UNKNOWN --format table "${tag}" + + + deploy: + needs: build + runs-on: ubuntu-latest + steps: + # Step 1: Checkout repository + - uses: actions/checkout@v4 + + # Step 2: Read image name + - name: Read docker name + id: yaml-docker-name + uses: jbutcher5/read-yaml@main + with: + file: 'build.yml' + key-path: '["docker_image_name"]' + + # Step 3: Read image version + - name: Read docker version + id: yaml-docker-version + uses: jbutcher5/read-yaml@main + with: + file: 'build.yml' + key-path: '["docker_image_version"]' + + # Step 4: Generate Docker tag + - name: Generate docker tag + env: + GITHUB_BRANCH: ${{ github.ref }} + docker_image_name: ${{ steps.yaml-docker-name.outputs.data }} + docker_image_version: ${{ steps.yaml-docker-version.outputs.data }} + run: | + branch_name=${GITHUB_BRANCH#refs/heads/} + echo "branch_name=${GITHUB_BRANCH#refs/heads/}" >> $GITHUB_ENV + echo "docker_tag=$docker_image_name:$docker_image_version" >> $GITHUB_ENV + echo "docker_tag_latest=$docker_image_name:latest" >> $GITHUB_ENV + docker_image_application=(${docker_image_name#*/}) + echo "docker_image_application=$docker_image_application" >> $GITHUB_ENV + echo "docker_image_version=$docker_image_version" >> $GITHUB_ENV + + # Step 5: Download Docker Image tar.gz Artifact + - name: Download Docker Image Artifact + uses: actions/download-artifact@v3 + with: + name: docker-image-tar + + # Step 6: Extract the Docker Image tar.gz + - name: Extract Docker Image tar.gz + run: | + tar -xzf "${docker_image_application}_${docker_image_version}.tar.gz" + + # Step 7: Load Docker Image + - name: Load Docker Image + run: | + docker load -i "${docker_image_application}_${docker_image_version}.tar" + + # Step 8: Log in to Docker Registry (use GitHub secrets for security) + - name: Login to Docker Registry + run: | + echo "${{ secrets.CR_PASSWORD }}" | docker login -u "${{ secrets.CR_USERNAME }}" --password-stdin "${{ secrets.CR_REGISTRY }}" + + # Step 9: Push Docker Image to Registry + - name: Push Docker Image to Registry + run: | + tag="${docker_image_application}:${docker_image_version}" + docker tag "${tag}" "${{ secrets.CR_REGISTRY }}"/"${{ secrets.CR_REPO }}"/"${tag}" + docker push "${{ secrets.CR_REGISTRY }}"/"${{ secrets.CR_REPO }}"/"${tag}" + + # Step 10: Login Docker Hub + - name: Login to Docker Hub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + + # Step 11: Push to Docker Hub + - name: push to Docker Hub + run: | + tag="${docker_image_application}:${docker_image_version}" + docker tag "${tag}" "docker.io/${{ env.docker_tag }}" + docker tag "${tag}" "docker.io/${{ env.docker_tag_latest }}" + docker push "docker.io/${{ env.docker_tag }}" + docker push "docker.io/${{ env.docker_tag_latest }}" \ No newline at end of file diff --git a/.github/workflows/build_publish.yml b/.github/workflows/build_publish.yml deleted file mode 100644 index c177e3f..0000000 --- a/.github/workflows/build_publish.yml +++ /dev/null @@ -1,60 +0,0 @@ -name: Build and Publish Docker - -on: - push: - branches: - - develop - - main -jobs: - build-and-publish-docker: - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v3 - - - name: Read docker name - id: yaml-docker-name - uses: jbutcher5/read-yaml@main - with: - file: 'build.yml' - key-path: '["docker_image_name"]' - - - name: Read docker version - id: yaml-docker-version - uses: jbutcher5/read-yaml@main - with: - file: 'build.yml' - key-path: '["docker_image_version"]' - - - name: Generate docker tag - env: - GITHUB_BRANCH: ${{ github.ref }} - docker_image_name: ${{ steps.yaml-docker-name.outputs.data }} - docker_image_version: ${{ steps.yaml-docker-version.outputs.data }} - run: | - branch_name=${GITHUB_BRANCH#refs/heads/} - echo "branch_name=${GITHUB_BRANCH#refs/heads/}" >> $GITHUB_ENV - if [[ "$branch_name" = "main" ]] - then - mType="" - else - mType="dev" - fi - echo "docker_tag=$docker_image_name:$mType$docker_image_version" >> $GITHUB_ENV - echo "docker_tag_latest=$docker_image_name:${mType}latest" >> $GITHUB_ENV - - - name: Login to Docker Hub - uses: docker/login-action@v2 - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} - - - name: Build and push - uses: docker/build-push-action@v4 - with: - context: . - push: true - no-cache: true - tags: | - ${{ env.docker_tag }} - ${{ env.docker_tag_latest }} diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index b31ea7b..0ebef06 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -4,7 +4,6 @@ on: branches: - develop - main - - secrets paths: # Only rebuild website when docs have changed - 'README.md' @@ -16,18 +15,13 @@ jobs: deploy: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - name: Checkout master + uses: actions/checkout@v2 - - name: Install Conda environment from environment.yml - uses: mamba-org/provision-with-micromamba@main + - name: Set up Python 3.x + uses: actions/setup-python@v2 with: - environment-file: docs/environment.yml - environment-name: env_zoo_calrissian - channels: terradue,eoepca,conda-forge - channel-priority: flexible - - - name: Install project - run: | - /home/runner/micromamba-root/envs/env_zoo_calrissian/bin/python setup.py install - - - run: /home/runner/micromamba-root/envs/env_zoo_calrissian/bin/mkdocs gh-deploy --force + python-version: 3.x + - run: | + pip install mkdocs-material mkdocs-mermaid2-plugin mkdocs-jupyter + mkdocs gh-deploy --force \ No newline at end of file diff --git a/.gitignore b/.gitignore index 5c29fa9..ec0db41 100644 --- a/.gitignore +++ b/.gitignore @@ -1,8 +1,8 @@ *.pyc __pycache__ -jupyterhub values.yaml *.egg-info build _README.md dist +.env-config-generator \ No newline at end of file diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 2172a56..f1de4c9 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -27,7 +27,7 @@ repos: args: - --max-line-length=88 - --max-doc-length=90 - - --ignore=E203,W503,W505 + - --ignore=E203,W503,W505,F821,E302,E402 - repo: https://github.com/psf/black rev: 22.3.0 hooks: diff --git a/Dockerfile b/Dockerfile index 87b350c..598c405 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM jupyterhub/k8s-hub:2.0.0 +FROM ghcr.io/eoepca/container-k8s-hub/container-k8s-hub:4.0.0 ARG NB_USER=johub ARG NB_UID=1001 @@ -6,35 +6,53 @@ ARG HOME=/home/johub USER root -RUN apt update && \ - apt install npm git sudo -y && \ - npm install -g configurable-http-proxy - -RUN adduser --disabled-password \ - --gecos "Default user" \ +# Packages update and dependencies installation +RUN microdnf update -y && \ + microdnf install -y \ + npm \ + git \ + sudo \ + python3-pip \ + python3-devel \ + gcc \ + libcurl-devel \ + openssl-devel \ + && microdnf clean all + +# Installation of configurable-http-proxy via npm +RUN npm install -g configurable-http-proxy + +# User creation +RUN adduser \ --uid ${NB_UID} \ --home ${HOME} \ - --force-badname \ - ${NB_USER} + ${NB_USER} \ + --comment "Default user" \ + --shell /bin/bash -RUN adduser jovyan sudo && \ - echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers +# Add jovyan to the sudoers group +RUN usermod -aG wheel jovyan && \ + echo '%wheel ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers +# Python packages installation from requirements.txt COPY requirements.txt /tmp/requirements.txt -RUN pip3 install --upgrade --no-cache-dir \ - setuptools \ - pip +RUN pip3 install --upgrade --no-cache-dir setuptools pip +# Specific Python dependencies installation RUN PYCURL_SSL_LIBRARY=openssl \ - pip install --no-cache-dir \ - -r /tmp/requirements.txt + pip install --no-cache-dir -r /tmp/requirements.txt + +# Check and correct requirejs version +RUN sed -i 's/"version": "[^"]*"/"version": "2.3.7"/' /usr/local/share/jupyterhub/static/components/requirejs/package.json -# So we can actually write a db file here +# Set permission on the directory /srv/jupyterhub RUN chown ${NB_USER}:${NB_USER} /srv/jupyterhub COPY . /tmp -RUN cd /tmp && python setup.py install +RUN cd /tmp && python3 setup.py install +# Set not root user USER ${NB_USER} -CMD ["jupyterhub", "--config", "/etc/jupyterhub/jupyterhub_config.py"] +# Command to start jupyterhub +CMD ["jupyterhub", "--config", "/etc/jupyterhub/jupyterhub_config.py"] \ No newline at end of file diff --git a/application_hub_context/__init__.py b/application_hub_context/__init__.py index e69de29..6225854 100644 --- a/application_hub_context/__init__.py +++ b/application_hub_context/__init__.py @@ -0,0 +1 @@ +version = "1.3.1" \ No newline at end of file diff --git a/application_hub_context/app_hub_context.py b/application_hub_context/app_hub_context.py index 3ebff7b..946a02a 100644 --- a/application_hub_context/app_hub_context.py +++ b/application_hub_context/app_hub_context.py @@ -1,10 +1,13 @@ import os import time +import yaml from abc import ABC from http import HTTPStatus from typing import Dict, TextIO +from jinja2 import Template from kubernetes import client, config +from kubernetes.utils import create_from_dict from kubernetes.client import Configuration from kubernetes.client.rest import ApiException from kubernetes.config.config_exception import ConfigException @@ -33,9 +36,10 @@ def __init__( self.api_client = self._get_api_client(self.kubeconfig_file) self.core_v1_api = self._get_core_v1_api() self.batch_v1_api = self._get_batch_v1_api() + self.apps_v1_api = self._get_apps_v1_api() self.rbac_authorization_v1_api = self._get_rbac_authorization_v1_api() + self.custom_objects_api = self._get_custom_objects_api() self.namespace = namespace - self.spawner = spawner # get the groups the user belongs to self.user_groups = [group.name for group in self.spawner.user.groups] @@ -46,7 +50,7 @@ def __init__( # loads config self.config_parser = ConfigParser.read_file( - config_path=config_path, user_groups=self.user_groups + config_path=config_path, user_groups=self.user_groups, spawner=self.spawner ) # update class dict with kwargs @@ -122,6 +126,12 @@ def _get_batch_v1_api(self) -> client.BatchV1Api: def _get_rbac_authorization_v1_api(self) -> client.RbacAuthorizationApi: return client.RbacAuthorizationV1Api(self.api_client) + def _get_apps_v1_api(self) -> client.AppsV1Api: + return client.AppsV1Api(self.api_client) + + def _get_custom_objects_api(self) -> client.CustomObjectsApi: + return client.CustomObjectsApi(self.api_client) + def is_object_created(self, read_method, **kwargs): read_methods = {} @@ -195,6 +205,40 @@ def initialise(self): def dispose(self): pass + def create_namespace( + self, labels: dict = None, annotations: dict = None + ) -> client.V1Namespace: + + if labels is None: + labels = self.spawner.user_namespace_labels + else: + labels = {**labels, **self.spawner.user_namespace_labels} + + if self.is_namespace_created(): + self.spawner.log.info( + f"namespace {self.namespace} exists, skipping creation" + ) + return self.core_v1_api.read_namespace(name=self.namespace) + + self.spawner.log.info(f"creating namespace {self.namespace}") + try: + body = client.V1Namespace( + metadata=client.V1ObjectMeta( + name=self.namespace, labels=labels, annotations=annotations + ) # noqa: E501 + ) + response = self.core_v1_api.create_namespace( + body=body, async_req=False + ) # noqa: E501 + + if not self.retry(self.is_namespace_created): + raise ApiException(http_resp=response) + self.spawner.log.info(f"namespace {self.namespace} created") + return response + except ApiException as e: + self.spawner.log.error(f"namespace {self.namespace} creation failed, {e}\n") + raise e + @staticmethod def retry(fun, max_tries=10, interval=1, **kwargs): for i in range(max_tries): @@ -390,7 +434,6 @@ def create_role( resources: list[str] = [""], api_groups: list[str] = ["*"], ): - if self.is_role_created(name=name): return self.rbac_authorization_v1_api.read_namespaced_role( name=name, namespace=self.namespace @@ -424,9 +467,7 @@ def create_role( raise e def create_image_pull_secret(self, name: str, data): - if self.is_image_pull_secret_created(name=name): - return self.core_v1_api.read_namespaced_secret( namespace=self.namespace, name=name ) # noqa: E501 @@ -500,11 +541,168 @@ def patch_service_account(self, secret_name: str): except ApiException as e: raise e + # new function to apply a set of manifests like kubectl apply -f + # def apply_manifests(self, manifest_file): + def apply_manifest(self, manifest): + + template = Template(yaml.dump(manifest)) + rendered_manifest = template.render(spawner=self.spawner) + + self.spawner.log.info(f"Applying manifest name: {yaml.safe_load(rendered_manifest).get('metadata').get('name')}") + + if yaml.safe_load(rendered_manifest)["kind"] in ["Release"]: + # see https://github.com/kubernetes-client/python/blob/master/kubernetes/docs/CustomObjectsApi.md#create_namespaced_custom_object + # Define the Crossplane HelmRelease details + group = "helm.crossplane.io" + version = "v1beta1" + plural = "releases" + + self.spawner.log.info(f"Creating Crossplane HelmRelease {yaml.safe_load(rendered_manifest).get('metadata').get('name')}") + + # Create the Crossplane HelmRelease + self.custom_objects_api.create_cluster_custom_object( + group=group, + version=version, + plural=plural, + body=yaml.safe_load(rendered_manifest) + ) + if yaml.safe_load(rendered_manifest)["kind"] in ["Object"]: + # see https://github.com/kubernetes-client/python/blob/master/kubernetes/docs/CustomObjectsApi.md#create_namespaced_custom_object + # Define the Crossplane Kubernetes Object details + group = "kubernetes.crossplane.io" + version = "v1alpha2" + plural = "objects" + + self.spawner.log.info(f"Creating Crossplane Kubernetes Object {yaml.safe_load(rendered_manifest).get('metadata').get('name')}") + + # Create the Crossplane Kubernetes Object + self.custom_objects_api.create_cluster_custom_object( + group=group, + version=version, + plural=plural, + body=yaml.safe_load(rendered_manifest) + ) + elif yaml.safe_load(rendered_manifest)["kind"] in ["ExternalSecret"]: + self.spawner.log.info(f"Creating ExternalSecret {yaml.safe_load(rendered_manifest).get('metadata').get('name')} in namespace {self.namespace}") + # Define the ExternalSecret details + group = "external-secrets.io" + version = "v1beta1" + namespace = f"jupyter-{self.spawner.user.name}" + plural = "externalsecrets" + + self.custom_objects_api.create_namespaced_custom_object( + group=group, + version=version, + namespace=namespace, + plural=plural, + body=yaml.safe_load(rendered_manifest) + ) + + elif yaml.safe_load(rendered_manifest)["kind"] in ["Secret"]: + + self.spawner.log.info(f"Creating Secret {yaml.safe_load(rendered_manifest).get('metadata').get('name')} in namespace {self.namespace}") + create_from_dict( + k8s_client=self.api_client, + data=yaml.safe_load(rendered_manifest), + verbose=True, + namespace=self.namespace, + ) + else: + self.spawner.log.info(f"Creating object of kind {yaml.safe_load(rendered_manifest).get('kind')} name {yaml.safe_load(rendered_manifest).get('metadata').get('name')} in namespace {self.namespace}") + create_from_dict( + k8s_client=self.api_client, + data=yaml.safe_load(rendered_manifest), + verbose=True, + namespace=self.namespace, + ) + + def unapply_manifests(self, manifest_content): + + for k8_object in manifest_content: + kind = k8_object.get("kind") + self.spawner.log.info( + f"Deleting {kind} {k8_object.get('metadata', {}).get('name')}" + ) + metadata = k8_object.get("metadata", {}) + namespace = metadata.get("namespace", self.namespace) + name = metadata.get("name") + + if not kind or not name: + continue + + try: + if kind == "Deployment": + self.apps_v1_api.delete_namespaced_deployment(name, namespace) + elif kind == "Service": + self.core_v1_api.delete_namespaced_service(name, namespace) + elif kind == "Job": + self.batch_v1_api.delete_namespaced_job(name, namespace) + elif kind == "Pod": + self.core_v1_api.delete_namespaced_pod(name, namespace) + elif kind == "Role": + self.rbac_authorization_v1_api.delete_namespaced_role(name, namespace) + elif kind == "RoleBinding": + self.rbac_authorization_v1_api.delete_namespaced_role_binding(name, namespace) + elif kind == "ServiceAccount": + self.core_v1_api.delete_namespaced_service_account(name, namespace) + elif kind == "ConfigMap": + self.core_v1_api.delete_namespaced_config_map(name, namespace) + elif kind == "Secret": + self.core_v1_api.delete_namespaced_secret(name, namespace) + elif kind == "Release": + self.spawner.log.info(f"Deleting Crossplane HelmRelease {name}") + response = self.custom_objects_api.delete_cluster_custom_object( + group="helm.crossplane.io", + version="v1beta1", + plural="releases", + name=name, + ) + self.spawner.log.info(f"Response: {response}") + elif kind == "Object": + self.spawner.log.info(f"Deleting Crossplane Kubernetes Object {name}") + response = self.custom_objects_api.delete_cluster_custom_object( + group="kubernetes.crossplane.io", + version="v1alpha2", + plural="objects", + name=name, + ) + self.spawner.log.info(f"Response: {response}") + elif kind == "ExternalSecret": + self.spawner.log.info(f"Deleting ExternalSecret {name} from namespace {namespace}") + self.custom_objects_api.delete_namespaced_custom_object( + group="external-secrets.io", + version="v1beta1", + namespace=namespace, + plural="externalsecrets", + name=name, + ) + # Add other kinds as needed + else: + self.spawner.log.error(f"Unsupported kind: {kind}") + except client.exceptions.ApiException as e: + self.spawner.log.error(f"An error occurred: {e}") + class DefaultApplicationHubContext(ApplicationHubContext): + + def __init__(self, namespace, spawner, config_path: str, kubeconfig_file: TextIO = None, skip_namespace_check=False, **kwargs): + + # skip_namespace_check is a flag to skip the namespace check and creation + self.skip_namespace_check = skip_namespace_check + + # add kwargs as members of the class + self.__dict__.update(kwargs) + + super().__init__(namespace, spawner, config_path, kubeconfig_file, **kwargs) + def get_profile_list(self): return self.config_parser.get_profiles() + # def render(self, manifest): + # # render the manifest using the spawner object + # template = Template(yaml.dump(manifest)) + # return yaml.safe_load(template.render(spawner=self.spawner)) + def initialise(self): # set the spawner timeout to 10 minutes self.spawner.http_timeout = 600 @@ -572,10 +770,44 @@ def initialise(self): profile_id=profile_id ) self.set_pod_env_vars(**(config_env_vars or {})) + + if not self.skip_namespace_check: + self.spawner.log.info(f"Checking namespace {self.namespace}") + # check the namespace + if not self.is_namespace_created(): + self.spawner.log.info(f"Creating namespace {self.namespace}") + self.create_namespace() + else: + self.spawner.log.info(f"Skipping namespace check") # process the config maps config_maps = self.config_parser.get_profile_config_maps(profile_id=profile_id) + # process the manifests + manifests = self.config_parser.get_profile_manifests(profile_id=profile_id) + + if manifests: + for manifest in manifests: + self.spawner.log.info(f"Apply manifest {manifest.name}") + + + for k8_object in manifest.content: + try: + + # Check and log the 'kind' of the Kubernetes object + if 'kind' in k8_object: + self.spawner.log.info(f"Applying manifest of kind: {k8_object['kind']}") + self.apply_manifest(k8_object) # Apply the manifest + else: + self.spawner.log.warning(f"Manifest does not contain a 'kind': {k8_object}") + + except Exception as err: + self.spawner.log.error(f"Unexpected {err}, {type(err)}") + self.spawner.log.error( + f"Skipping creation of manifest {manifest.name}" + ) + + if config_maps: for config_map in config_maps: try: @@ -588,25 +820,34 @@ def initialise(self): annotations=None, labels=None, ) - self.spawner.log.info(f"Mounting configmap {config_map.name}") - self.spawner.volume_mounts.extend( - [ - { - "name": config_map.name, - "mountPath": config_map.mount_path, - "subPath": config_map.key, - }, - ] - ) + if config_map.mount_path is not None: - self.spawner.volumes.extend( - [ - { - "name": config_map.name, - "configMap": {"name": config_map.key}, - } - ] - ) + self.spawner.log.info(f"Mounting configmap {config_map.name}") + self.spawner.volume_mounts.extend( + [ + { + "name": config_map.name, + "mountPath": config_map.mount_path, + "subPath": config_map.key, + }, + ] + ) + self.spawner.volumes.extend( + [ + { + "name": config_map.name, + "configMap": { + "name": config_map.name, + "defaultMode": int(config_map.default_mode, 8) + if config_map.default_mode + else 0o644, # noqa: E501 + }, + } + ] + ) + self.spawner.log.info( + f"Mounted configmap {config_map.name} (key {config_map.key})" # noqa: E501 + ) except Exception as err: self.spawner.log.error(f"Unexpected {err=}, {type(err)=}") self.spawner.log.error( @@ -668,7 +909,6 @@ def initialise(self): try: # checking if role binding is already created if not self.is_role_binding_created(name=role_binding.name): - # checking if role is already created if not self.is_role_created(name=role_binding.role.name): self.spawner.log.info( @@ -720,7 +960,9 @@ def initialise(self): "Skipping creation of image pull secret" f" {image_pull_secret.name}" ) - + self.spawner.log.info( + f"Patch service account with image pull secret {image_pull_secret.name}" + ) self.patch_service_account(secret_name=image_pull_secret.name) # process the init containers @@ -755,6 +997,73 @@ def initialise(self): f"Skipping creation of init container {init_container.name}" ) + + # process the pod env vars from config maps + env_from_config_maps = self.config_parser.get_profile_env_from_config_maps( + profile_id=profile_id + ) + self.spawner.log.info(f"env_from_config_maps {env_from_config_maps}") + if env_from_config_maps: + self.spawner.extra_container_config["env_from"] = [] + + for env_from_config_map in env_from_config_maps: + self.spawner.log.info(f"env_from_config_map {env_from_config_map}") + self.spawner.extra_container_config["env_from"].append( + { + "configMapRef": { + "name": env_from_config_map, + }}) + self.spawner.log.info(f"extra_container_config {self.spawner.extra_container_config}") + + # process the pod env vars from secrets + env_from_secrets = self.config_parser.get_profile_env_from_secrets( + profile_id=profile_id + ) + self.spawner.log.info(f"env_from_secrets {env_from_secrets}") + if env_from_secrets: + if self.spawner.extra_container_config["env_from"] is None: + self.spawner.extra_container_config["env_from"] = [] + + for env_from_secret in env_from_secrets: + self.spawner.log.info(f"env_from_secret {env_from_secret}") + self.spawner.extra_container_config["env_from"].append( + { + "secretRef": { + "name": env_from_secret, + }}) + self.spawner.log.info(f"extra_container_config {self.spawner.extra_container_config}") + + + secret_mounts = self.config_parser.get_profile_secret_mounts( + profile_id=profile_id + ) + + if secret_mounts: + for secret_mount in secret_mounts: + self.spawner.log.info(f"Mounting secret {secret_mount.name}") + self.spawner.volume_mounts.extend( + [ + { + "name": secret_mount.name, + "mountPath": secret_mount.mount_path, + "subPath": secret_mount.sub_path, + }, + ] + ) + + self.spawner.volumes.extend( + [ + { + "name": secret_mount.name, + "secret": { + "secretName": secret_mount.name, + }, + } + ] + ) + + self.spawner.log.info(f"Mounted secret {secret_mount.name}") + def dispose(self): profile_id = self.config_parser.get_profile_by_slug(slug=self.profile_slug).id @@ -794,6 +1103,17 @@ def dispose(self): self.spawner.log.info(f"Dispose role binding {role_binding.name}") self.delete_role_binding(role_binding=role_binding) + # process the manifests + manifests = self.config_parser.get_profile_manifests(profile_id=profile_id) + + if manifests: + for manifest in manifests: + if manifest.persist: + self.spawner.log.info(f"Persist manifest {manifest.name}") + if not manifest.persist: + self.spawner.log.info(f"Un-apply manifest {manifest.name}") + self.unapply_manifests(manifest_content=manifest.content) + # deal with the image pull secrets image_pull_secrets = self.config_parser.get_profile_image_pull_secrets( profile_id=profile_id @@ -814,7 +1134,6 @@ def dispose(self): ) for elem in service_account_body.image_pull_secrets: if elem.name == image_pull_secret.name: - service_account_body.image_pull_secrets.remove( {"name": elem.name} ) diff --git a/application_hub_context/models.py b/application_hub_context/models.py index ec03106..86b52c8 100644 --- a/application_hub_context/models.py +++ b/application_hub_context/models.py @@ -1,5 +1,5 @@ from enum import Enum -from typing import List, Optional, Union +from typing import Dict, List, Optional, Union from pydantic import BaseModel, Field @@ -37,6 +37,13 @@ class defining a volume object: persist: bool +class Manifest(BaseModel): + name: str + key: str + content: Optional[List[Dict]] = None + persist: Optional[bool] = True + + class ConfigMap(BaseModel): """ @@ -50,8 +57,8 @@ class ConfigMap(BaseModel): name: str key: str - mount_path: str - default_mode: Optional[str] + mount_path: Optional[str] = None + default_mode: Optional[str] = None readonly: bool content: Optional[str] = None persist: Optional[bool] = True @@ -67,6 +74,10 @@ class KubespawnerOverride(BaseModel): extra_resource_guarantees: Optional[dict] = {} +class InitContainerVolumeMount(VolumeMount): + sub_path: str + + class InitContainer(BaseModel): name: str image: str @@ -152,6 +163,10 @@ class ImagePullSecret(BaseModel): persist: bool = True data: Optional[str] = None +class SecretMount(BaseModel): + name: str + mount_path: str + sub_path: Optional[str] = None class Profile(BaseModel): id: str @@ -165,7 +180,10 @@ class Profile(BaseModel): role_bindings: Optional[List[RoleBinding]] = None image_pull_secrets: Optional[List[ImagePullSecret]] = None init_containers: Optional[List[InitContainer]] = None - + manifests: Optional[List[Manifest]] = None + env_from_config_maps: Optional[List[str]] = None + env_from_secrets: Optional[List[str]] = None + secret_mounts: Optional[List[SecretMount]] = None class Config(BaseModel): profiles: List[Profile] diff --git a/application_hub_context/parser.py b/application_hub_context/parser.py index 8ed1e44..679c3b1 100644 --- a/application_hub_context/parser.py +++ b/application_hub_context/parser.py @@ -1,5 +1,5 @@ import yaml - +from jinja2 import Template from application_hub_context.models import Config undefined = object() @@ -12,13 +12,24 @@ def __init__(self, config_data, user_groups): self.user_groups = user_groups @classmethod - def read_file(cls, config_path, user_groups): + def read_file(cls, config_path, user_groups, spawner): """reads a config file encoded in YAML""" with open(config_path, "r") as stream: try: - config_data = yaml.safe_load(stream) + # Read the file as a raw string + raw_content = stream.read() + + # Render the content as a Jinja2 template + template = Template(raw_content) + rendered_content = template.render(spawner=spawner) + + # Parse the rendered content as YAML + config_data = yaml.safe_load(rendered_content) + except yaml.YAMLError as exc: - print(exc) + print(f"YAML Error: {exc}") + except Exception as e: + print(f"Error: {e}") return cls(config_data=config_data, user_groups=user_groups) @@ -124,3 +135,31 @@ def get_profile_image_pull_secrets(self, profile_id): def get_profile_init_containers(self, profile_id): """returns the image pull secrets""" return self.get_profile_by_id(profile_id=profile_id).init_containers + + def get_profile_manifests(self, profile_id): + """returns the profile manifests""" + try: + return self.get_profile_by_id(profile_id=profile_id).manifests + except AttributeError: + pass + + def get_profile_env_from_config_maps(self, profile_id): + """returns the profile env from config maps""" + try: + return self.get_profile_by_id(profile_id=profile_id).env_from_config_maps + except AttributeError: + pass + + def get_profile_env_from_secrets(self, profile_id): + """returns the profile env from secrets""" + try: + return self.get_profile_by_id(profile_id=profile_id).env_from_secrets + except AttributeError: + pass + + def get_profile_secret_mounts(self, profile_id): + """returns the profile secret mounts""" + try: + return self.get_profile_by_id(profile_id=profile_id).secret_mounts + except AttributeError: + pass \ No newline at end of file diff --git a/build.yml b/build.yml index ef8ccfc..5e78c85 100644 --- a/build.yml +++ b/build.yml @@ -1,2 +1,2 @@ docker_image_name: eoepca/application-hub -docker_image_version: 1.2.0 +docker_image_version: 1.4.0 diff --git a/config-generator/README.md b/config-generator/README.md new file mode 100644 index 0000000..8a4a3c1 --- /dev/null +++ b/config-generator/README.md @@ -0,0 +1,5 @@ +## ApplicationHub configuration generator + +This folder contains a notebook and the python modules to support the generation of ApplicationHub configurations. + +Create a Python environment with the dependencies listed in the file `requirements.txt` and open the notebook `config-generator.ipynb` \ No newline at end of file diff --git a/config-generator/__init__.py b/config-generator/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/config-generator/config-generator-eoepca-demo.ipynb b/config-generator/config-generator-eoepca-demo.ipynb new file mode 100644 index 0000000..57905c8 --- /dev/null +++ b/config-generator/config-generator-eoepca-demo.ipynb @@ -0,0 +1,710 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import yaml\n", + "from models import *\n", + "import os" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Configuration\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "storage_class_rwo = \"managed-nfs-storage\"\n", + "storage_class_rwx = \"managed-nfs-storage\"\n", + "\n", + "workspace_volume_size = \"50Gi\"\n", + "calrissian_volume_size = \"50Gi\"\n" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Volumes\n", + "\n", + "Create the Volumes" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Workspace Volume\n", + "\n", + "The workspace volume is persisted." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "workspace_volume = Volume(\n", + " name=\"workspace-volume\",\n", + " size=workspace_volume_size,\n", + " claim_name=\"workspace-claim\",\n", + " mount_path=\"/workspace\",\n", + " storage_class=storage_class_rwo,\n", + " access_modes=[\"ReadWriteOnce\"],\n", + " volume_mount=VolumeMount(name=\"workspace-volume\", mount_path=\"/workspace\"),\n", + " persist=False,\n", + ")" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Calrissian Volume\n", + "\n", + "This is a RWX volume, not persisted" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "calrissian_volume = Volume(\n", + " name=\"calrissian-volume\",\n", + " claim_name=\"calrissian-claim\",\n", + " size=calrissian_volume_size,\n", + " storage_class=storage_class_rwx,\n", + " access_modes=[\"ReadWriteMany\"],\n", + " volume_mount=VolumeMount(name=\"calrissian-volume\", mount_path=\"/calrissian\"),\n", + " persist=False,\n", + ")" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## ConfigMaps\n", + "\n", + "These configmaps are mounted as files on the pod." + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### bash login\n", + "\n", + "This file is used for the JupyterLab Terminal configuration" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "with open(\"config-maps/bash-login\", \"r\") as f:\n", + " content = f.read()\n", + "\n", + "bash_login_cm = ConfigMap(\n", + " name=\"bash-login\",\n", + " key=\"bash-login\",\n", + " content=content,\n", + " readonly=True,\n", + " persist=False,\n", + " mount_path=\"/workspace/.bash_login\",\n", + ")" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### bash.rc\n" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "with open(\"config-maps/bash-rc\", \"r\") as f:\n", + " content = f.read()\n", + "bash_rc_cm = ConfigMap(\n", + " name=\"bash-rc\",\n", + " key=\"bash-rc\",\n", + " content=content,\n", + " readonly=True,\n", + " persist=False,\n", + " mount_path=\"/workspace/.bashrc\",\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## bash login\n" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [], + "source": [ + "with open(\"config-maps/bash-login\", \"r\") as f:\n", + " content = f.read()\n", + "\n", + "bash_login_cm = ConfigMap(\n", + " name=\"bash-login\",\n", + " key=\"bash-login\",\n", + " content=content,\n", + " readonly=True,\n", + " persist=False,\n", + " mount_path=\"/workspace/.bash_login\",\n", + ")" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Profiles" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [], + "source": [ + "profiles = []" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Coder" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [], + "source": [ + "coders = {\n", + " \"coder1\": {\n", + " \"display_name\": \"Code Server Small\",\n", + " \"slug\": \"ellip_studio_coder_slug_s\",\n", + " \"cpu_limit\": 2,\n", + " \"mem_limit\": \"8G\",\n", + " },\n", + " \"coder2\": {\n", + " \"display_name\": \"Code Server Medium\",\n", + " \"slug\": \"ellip_studio_coder_slug_m\",\n", + " \"cpu_limit\": 4,\n", + " \"mem_limit\": \"12G\",\n", + " },\n", + "}\n", + "\n", + "for key, value in coders.items():\n", + " coder_definition = ProfileDefinition(\n", + " display_name=value[\"display_name\"],\n", + " slug=value[\"slug\"],\n", + " default=False,\n", + " kubespawner_override=KubespawnerOverride(\n", + " cpu_limit=value[\"cpu_limit\"],\n", + " mem_limit=value[\"mem_limit\"],\n", + " image=\"eoepca/pde-code-server:develop\",\n", + " ),\n", + " )\n", + "\n", + " coder_profile = Profile(\n", + " id=f\"profile_studio_{key}\",\n", + " groups=[\"group-a\", \"group-b\"],\n", + " definition=coder_definition,\n", + " node_selector={},\n", + " volumes=[calrissian_volume, workspace_volume],\n", + " config_maps=[\n", + " bash_rc_cm,\n", + " ],\n", + " pod_env_vars={\n", + " \"HOME\": \"/workspace\",\n", + " \"CONDA_ENVS_PATH\": \"/workspace/.envs\",\n", + " },\n", + " )\n", + "\n", + " profiles.append(coder_profile)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## init.sh script" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [], + "source": [ + "with open(\"./config-maps/init.sh\", \"r\") as f:\n", + " content = f.read()\n", + "\n", + "init_cm = ConfigMap(\n", + " name=\"init\",\n", + " key=\"init\",\n", + " content=content,\n", + " readonly=True,\n", + " persist=False,\n", + " mount_path=\"/opt/init/.init.sh\",\n", + " default_mode=\"0660\",\n", + ")\n", + "\n", + "\n", + "init_context_volume_mount = InitContainerVolumeMount(\n", + " mount_path=\"/opt/init/.init.sh\", name=\"init\", sub_path=\"init\"\n", + ")\n", + "init_container = InitContainer(\n", + " name=\"init-file-on-volume\",\n", + " image=\"eoepca/pde-code-server:develop\",\n", + " command=[\"sh\", \"-c\", \"sh /opt/init/.init.sh\"],\n", + " volume_mounts=[\n", + " VolumeMount(name=\"workspace-volume\", mount_path=\"/workspace\"),\n", + " init_context_volume_mount,\n", + " ],\n", + ")\n", + "\n", + "eoepca_demo_init_script_profile = Profile(\n", + " id=f\"profile_demo_init_script\",\n", + " groups=[\"group-a\", \"group-b\"],\n", + " definition=ProfileDefinition(\n", + " display_name=\"Coder demo init script\",\n", + " description=\"This profile is used to demonstrate the use of an init script\",\n", + " slug=\"eoepca_demo_init_script\",\n", + " default=False,\n", + " kubespawner_override=KubespawnerOverride(\n", + " cpu_guarantee=1,\n", + " cpu_limit=2,\n", + " mem_guarantee=\"4G\",\n", + " mem_limit=\"6G\",\n", + " image=\"eoepca/pde-code-server:develop\",\n", + " ),\n", + " ),\n", + " node_selector={},\n", + " volumes=[calrissian_volume, workspace_volume],\n", + " config_maps=[init_cm],\n", + " pod_env_vars={\n", + " \"HOME\": \"/workspace\",\n", + " \"CONDA_ENVS_PATH\": \"/workspace/.envs\",\n", + " \"CONDARC\": \"/workspace/.condarc\",\n", + " \"XDG_RUNTIME_DIR\": \"/workspace/.local\",\n", + " \"CODE_SERVER_WS\": \"/workspace/mastering-app-package\",\n", + " },\n", + " init_containers=[init_container],\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [], + "source": [ + "profiles.append(eoepca_demo_init_script_profile)\n", + "\n", + "eoepca_demo_init_script_profile = Profile(\n", + " id=f\"profile_demo_init_script\",\n", + " groups=[\"group-a\", \"group-b\"],\n", + " definition=ProfileDefinition(\n", + " display_name=\"Coder demo init script\",\n", + " description=\"This profile is used to demonstrate the use of an init script\",\n", + " slug=\"eoepca_demo_init_script\",\n", + " default=False,\n", + " kubespawner_override=KubespawnerOverride(\n", + " cpu_guarantee=1,\n", + " cpu_limit=2,\n", + " mem_guarantee=\"4G\",\n", + " mem_limit=\"6G\",\n", + " image=\"eoepca/pde-code-server:develop\",\n", + " ),\n", + " ),\n", + " node_selector={},\n", + " volumes=[calrissian_volume, workspace_volume],\n", + " config_maps=[init_cm],\n", + " pod_env_vars={\n", + " \"HOME\": \"/workspace\",\n", + " \"CONDA_ENVS_PATH\": \"/workspace/.envs\",\n", + " \"CONDARC\": \"/workspace/.condarc\",\n", + " \"XDG_RUNTIME_DIR\": \"/workspace/.local\",\n", + " \"CODE_SERVER_WS\": \"/workspace/mastering-app-package\",\n", + " },\n", + " init_containers=[init_container],\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## JupyterLab" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [], + "source": [ + "image = \"jupyter/scipy-notebook\"\n", + "\n", + "\n", + "eoepca_jupyter_lab_profile = Profile(\n", + " id=\"profile_jupyter_lab\",\n", + " groups=[\"group-c\"],\n", + " definition=ProfileDefinition(\n", + " display_name=\"Jupyter Lab\",\n", + " description=\"Jupyter Lab with Python 3.11\",\n", + " slug=\"eoepca_jupyter_lab\",\n", + " default=False,\n", + " kubespawner_override=KubespawnerOverride(\n", + " cpu_guarantee=1,\n", + " cpu_limit=2,\n", + " mem_guarantee=\"4G\",\n", + " mem_limit=\"6G\",\n", + " image=image,\n", + " ),\n", + " ),\n", + " node_selector={},\n", + " volumes=[workspace_volume],\n", + " config_maps=[],\n", + " pod_env_vars={\n", + " \"HOME\": \"/workspace\",\n", + " \"XDG_RUNTIME_DIR\": \"/workspace/.local\",\n", + " \"XDG_CONFIG_HOME\": \"/workspace/.config\",\n", + " },\n", + ")\n", + "\n", + "profiles.append(eoepca_jupyter_lab_profile)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Image pull secret\n" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [], + "source": [ + "image_pull_secret = ImagePullSecret(\n", + " name=\"cr-config\",\n", + " persist=False,\n", + " data=\"ewogICAgImF1dGhzIjogewogICAgICAgICJjci50ZXJyYWR1ZS5jb20iOiB7CiAgICAgICAgICAgICJ1c2VybmFtZSI6ICJyb2JvdCRlb2VwY2EtcGx1cy1ybyIsCiAgICAgICAgICAgICJwYXNzd29yZCI6ICJQMlE4TnkyZ0lHODhkZkxveXlLN05QVUZVbHJOekFZSiIsCiAgICAgICAgICAgICJlbWFpbCI6ICJlb2VwY2EtcGx1c0B0ZXJyYWR1ZS5jb20iLAogICAgICAgICAgICAiYXV0aCI6ICJjbTlpYjNRa1pXOWxjR05oTFhCc2RYTXRjbTg2VURKUk9FNTVNbWRKUnpnNFpHWk1iM2w1U3pkT1VGVkdWV3h5VG5wQldVbz0iCiAgICAgICAgfQogICAgfQp9\",\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [], + "source": [ + "image = \"cr.terradue.com/eoepca-plus/scipy-notebook@sha256:f339a9fa98d3d0c1fa8d7cc850e7f5a46845781f49bee86aacba059669d02d54\"\n", + "image = \"eoepca/iat-jupyterlab:develop\"\n", + "\n", + "eoepca_jupyter_lab_profile_2 = Profile(\n", + " id=\"profile_jupyter_lab_2\",\n", + " groups=[\"group-c\"],\n", + " definition=ProfileDefinition(\n", + " display_name=\"Jupyter Lab - profile 2\",\n", + " description=\"Jupyter Lab with Python 3.11 private image - demoes the use of an image pull secret\",\n", + " slug=\"eoepca_jupyter_lab_2\",\n", + " default=False,\n", + " kubespawner_override=KubespawnerOverride(\n", + " cpu_guarantee=1,\n", + " cpu_limit=2,\n", + " mem_guarantee=\"4G\",\n", + " mem_limit=\"6G\",\n", + " image=image,\n", + " ),\n", + " ),\n", + " node_selector={},\n", + " volumes=[workspace_volume],\n", + " config_maps=[],\n", + " pod_env_vars={\n", + " \"HOME\": \"/workspace\",\n", + " \"XDG_RUNTIME_DIR\": \"/workspace/.local\",\n", + " \"XDG_CONFIG_HOME\": \"/workspace/.config\",\n", + " },\n", + " image_pull_secrets=[image_pull_secret],\n", + ")\n", + "\n", + "profiles.append(eoepca_jupyter_lab_profile_2)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Stage-in/out" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [], + "source": [ + "with open(\"./config-maps/init-stac.sh\", \"r\") as f:\n", + " content = f.read()\n", + "\n", + "init_cm = ConfigMap(\n", + " name=\"init\",\n", + " key=\"init\",\n", + " content=content,\n", + " readonly=True,\n", + " persist=False,\n", + " mount_path=\"/opt/init/.init.sh\",\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [], + "source": [ + "with open(\"manifests/manifest.yaml\", \"r\") as f:\n", + " content = yaml.safe_load_all(f.read())\n", + "\n", + "\n", + "\n", + "localstack_manifest = Manifest(\n", + " name=\"manifests\", key=\"manifests\", readonly=True, persist=False, content=[e for e in content]\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## e-learning" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [], + "source": [ + "image = \"docker.io/eoepca/pde-code-server@sha256:f57a3d5eabcae667e0db6e84a57b0c07c692c88f0fb5c8f6900ab8d5e38fcd40\"\n", + "\n", + "coder_profile_stac = Profile(\n", + " id=f\"profile_studio_coder_stac\",\n", + " groups=[\"group-a\", \"group-b\"],\n", + " definition=ProfileDefinition(\n", + " display_name=\"Understanding STAC for input/output data modelling\",\n", + " description=\"Understand the role of STAC in input/output data manifests in EO data processing workflows\",\n", + " slug=\"eoepca_coder_slug_stac\",\n", + " default=False,\n", + " kubespawner_override=KubespawnerOverride(\n", + " cpu_guarantee=1,\n", + " cpu_limit=2,\n", + " mem_guarantee=\"4G\",\n", + " mem_limit=\"6G\",\n", + " image=image,\n", + " ),\n", + " ),\n", + " node_selector={},\n", + " volumes=[workspace_volume],\n", + " config_maps=[init_cm, bash_rc_cm, bash_login_cm],\n", + " pod_env_vars={\n", + " \"HOME\": \"/workspace\",\n", + " \"CONDA_ENVS_PATH\": \"/workspace/.envs\",\n", + " \"CONDARC\": \"/workspace/.condarc\",\n", + " \"XDG_RUNTIME_DIR\": \"/workspace/.local\",\n", + " \"XDG_RUNTIME_DIR\": \"/workspace/.local\",\n", + " \"XDG_CONFIG_HOME\": \"/workspace/.local\",\n", + " \"XDG_DATA_HOME\": \"/workspace/.local/share/\",\n", + " \"CWLTOOL_OPTIONS\": \"--podman\", \n", + " \"CODE_SERVER_WS\": \"/workspace/stac-eoap\",\n", + " \"AWS_DEFAULT_REGION\": \"us-east-1\",\n", + " \"AWS_ACCESS_KEY_ID\": \"test\",\n", + " \"AWS_SECRET_ACCESS_KEY\": \"test\",\n", + " },\n", + " role_bindings=[],\n", + " init_containers=[init_container],\n", + " image_pull_secrets=[image_pull_secret],\n", + " manifests=[localstack_manifest],\n", + ")\n", + "\n", + "profiles.append(coder_profile_stac)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## QGIS" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [], + "source": [ + "with open(\"./config-maps/init-qgis.sh\", \"r\") as f:\n", + " content = f.read()\n", + "\n", + "init_qgis_cm = ConfigMap(\n", + " name=\"init\",\n", + " key=\"init\",\n", + " content=content,\n", + " readonly=True,\n", + " persist=False,\n", + " mount_path=\"/opt/init/.init.sh\",\n", + ")\n", + "\n", + "image = \"eoepca/iga-remote-desktop-qgis:1.1.3\"\n", + "\n", + "qgis_profile = Profile(\n", + " id=\"profile_studio_desktop_qgis\",\n", + " groups=[\"group-a\", \"group-b\"],\n", + " definition=ProfileDefinition(\n", + " display_name=\"QGIS on a Remote Desktop\",\n", + " description=\"Spatial visualization and decision-making tools for everyone\",\n", + " slug=\"eoepca_desktop_qgis\",\n", + " default=False,\n", + " kubespawner_override=KubespawnerOverride(\n", + " cpu_limit=2,\n", + " mem_limit=\"2G\",\n", + " image=image,\n", + " ),\n", + " ),\n", + " node_selector={},\n", + " volumes=[workspace_volume],\n", + " config_maps=[bash_rc_cm, bash_login_cm],\n", + " pod_env_vars={\n", + " \"HOME\": \"/workspace\"\n", + " },\n", + " default_url=\"desktop\",\n", + " init_containers=[]\n", + " )\n", + "\n", + "profiles.append(qgis_profile)" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Configuration" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [], + "source": [ + "config = Config(\n", + " profiles=profiles\n", + ")\n", + "\n", + "with open(\n", + " \"../eoepca-demo/config.yml\", \"w\"\n", + ") as file:\n", + " yaml.dump(config.dict(), file)" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": {}, + "outputs": [], + "source": [ + "# profiles" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": ".env-config-generator", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.14" + }, + "orig_nbformat": 4 + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/config-generator/config-generator.ipynb b/config-generator/config-generator.ipynb new file mode 100644 index 0000000..9afb17b --- /dev/null +++ b/config-generator/config-generator.ipynb @@ -0,0 +1,573 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import yaml\n", + "from models import *\n", + "import os" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "with open(\"manifests/manifest.yaml\", \"r\") as f:\n", + " content = yaml.safe_load_all(f.read())\n", + "\n", + "\n", + "\n", + "localstack_manifest = Manifest(\n", + " name=\"manifests\", key=\"manifests\", readonly=True, persist=False, content=[e for e in content]\n", + ")" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Configuration\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "storage_class_rwo = \"standard\"\n", + "storage_class_rwx = \"standard\"\n", + "\n", + "workspace_volume_size = \"50Gi\"\n", + "calrissian_volume_size = \"50Gi\"\n", + "\n", + "node_selector = {} #\"k8s.scaleway.com/pool-name\": \"application-hub\"}\n" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Volumes\n", + "\n", + "Create the Volumes" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Workspace Volume\n", + "\n", + "The workspace volume is persisted." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "workspace_volume = Volume(\n", + " name=\"workspace-volume\",\n", + " size=workspace_volume_size,\n", + " claim_name=\"workspace-claim\",\n", + " mount_path=\"/workspace\",\n", + " storage_class=storage_class_rwo,\n", + " access_modes=[\"ReadWriteOnce\"],\n", + " volume_mount=VolumeMount(name=\"workspace-volume\", mount_path=\"/workspace\"),\n", + " persist=True,\n", + ")" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Calrissian Volume\n", + "\n", + "This is a RWX volume, not persisted" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "calrissian_volume = Volume(\n", + " name=\"calrissian-volume\",\n", + " claim_name=\"calrissian-claim\",\n", + " size=calrissian_volume_size,\n", + " storage_class=storage_class_rwx,\n", + " access_modes=[\"ReadWriteMany\"],\n", + " volume_mount=VolumeMount(name=\"calrissian-volume\", mount_path=\"/calrissian\"),\n", + " persist=False,\n", + ")" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## ConfigMaps\n", + "\n", + "These configmaps are mounted as files on the pod." + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### bash login\n", + "\n", + "This file is used for the JupyterLab Terminal configuration" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "with open(\"config-maps/bash-login\", \"r\") as f:\n", + " content = f.read()\n", + "\n", + "bash_login_cm = ConfigMap(\n", + " name=\"bash-login\",\n", + " key=\"bash-login\",\n", + " content=content,\n", + " readonly=True,\n", + " persist=True,\n", + " mount_path=\"/workspace/.bash_login\",\n", + ")" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### bash.rc\n" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [], + "source": [ + "with open(\"config-maps/bash-rc\", \"r\") as f:\n", + " content = f.read()\n", + "bash_rc_cm = ConfigMap(\n", + " name=\"bash-rc\",\n", + " key=\"bash-rc\",\n", + " content=content,\n", + " readonly=True,\n", + " persist=True,\n", + " mount_path=\"/workspace/.bashrc\",\n", + ")" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Profiles" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [], + "source": [ + "profiles = []" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Coder" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [], + "source": [ + "coders = {\n", + " \"coder1\": {\n", + " \"display_name\": \"Code Server Small\",\n", + " \"slug\": \"eoepca_coder_slug_s\",\n", + " \"cpu_limit\": 2,\n", + " \"mem_limit\": \"8G\",\n", + " },\n", + " \"coder2\": {\n", + " \"display_name\": \"Code Server Medium\",\n", + " \"slug\": \"eoepca_coder_slug_m\",\n", + " \"cpu_limit\": 4,\n", + " \"mem_limit\": \"12G\",\n", + " },\n", + "}\n", + "\n", + "for key, value in coders.items():\n", + " coder_definition = ProfileDefinition(\n", + " display_name=value[\"display_name\"],\n", + " slug=value[\"slug\"],\n", + " default=False,\n", + " kubespawner_override=KubespawnerOverride(\n", + " cpu_limit=value[\"cpu_limit\"],\n", + " mem_limit=value[\"mem_limit\"],\n", + " image=\"eoepca/pde-code-server:develop\",\n", + " ),\n", + " )\n", + "\n", + " coder_profile = Profile(\n", + " id=f\"profile_studio_{key}\",\n", + " groups=[\"group-a\", \"group-b\"],\n", + " definition=coder_definition,\n", + " node_selector=node_selector,\n", + " volumes=[calrissian_volume, workspace_volume],\n", + " config_maps=[\n", + " bash_rc_cm,\n", + " ],\n", + " pod_env_vars={\n", + " \"HOME\": \"/workspace\",\n", + " \"CONDA_ENVS_PATH\": \" /workspace/.envs\",\n", + " },\n", + " manifests=[localstack_manifest],\n", + " env_from_config_maps=[\"my-config\"],\n", + " env_from_secrets=[\"my-secret\", \"data-by-name\"],\n", + " secret_mounts=[SecretMount(name=\"aws-credentials-{{ spawner.user.name }}\", mount_path=\"/workspace/.aws\"), \n", + " SecretMount(name=\"data-by-name\", mount_path=\"/workspace/.data-by-name\"), \n", + " SecretMount(name=\"eoepca-plus-secret-ro\", mount_path=\"/workspace/.docker\"),],\n", + " image_pull_secrets=[ImagePullSecret(name=\"eoepca-plus-secret-ro\")],\n", + " )\n", + "\n", + " profiles.append(coder_profile)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## init.sh script" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [], + "source": [ + "with open(\"./config-maps/init.sh\", \"r\") as f:\n", + " content = f.read()\n", + "\n", + "init_cm = ConfigMap(\n", + " name=\"init\",\n", + " key=\"init\",\n", + " content=content,\n", + " readonly=True,\n", + " persist=False,\n", + " mount_path=\"/opt/init/.init.sh\",\n", + " default_mode=\"0660\",\n", + ")\n", + "\n", + "\n", + "init_context_volume_mount = InitContainerVolumeMount(\n", + " mount_path=\"/opt/init/.init.sh\", name=\"init\", sub_path=\"init\"\n", + ")\n", + "init_container = InitContainer(\n", + " name=\"init-file-on-volume\",\n", + " image=\"eoepca/pde-code-server:develop\",\n", + " command=[\"sh\", \"-c\", \"sh /opt/init/.init.sh\"],\n", + " volume_mounts=[\n", + " VolumeMount(name=\"workspace-volume\", mount_path=\"/workspace\"),\n", + " init_context_volume_mount,\n", + " ],\n", + ")\n", + "\n", + "eoepca_demo_init_script_profile = Profile(\n", + " id=f\"profile_demo_init_script\",\n", + " groups=[\"group-a\", \"group-b\"],\n", + " definition=ProfileDefinition(\n", + " display_name=\"Coder demo init script\",\n", + " description=\"This profile is used to demonstrate the use of an init script\",\n", + " slug=\"eoepca_demo_init_script\",\n", + " default=False,\n", + " kubespawner_override=KubespawnerOverride(\n", + " cpu_guarantee=1,\n", + " cpu_limit=2,\n", + " mem_guarantee=\"4G\",\n", + " mem_limit=\"6G\",\n", + " image=\"eoepca/pde-code-server:develop\",\n", + " ),\n", + " ),\n", + " node_selector=node_selector,\n", + " volumes=[calrissian_volume, workspace_volume],\n", + " config_maps=[init_cm],\n", + " pod_env_vars={\n", + " \"HOME\": \"/workspace\",\n", + " \"CONDA_ENVS_PATH\": \"/workspace/.envs\",\n", + " \"CONDARC\": \"/workspace/.condarc\",\n", + " \"XDG_RUNTIME_DIR\": \"/workspace/.local\",\n", + " \"CODE_SERVER_WS\": \"/workspace/mastering-app-package\",\n", + " },\n", + " init_containers=[init_container],\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [], + "source": [ + "profiles.append(eoepca_demo_init_script_profile)\n", + "\n", + "eoepca_demo_init_script_profile = Profile(\n", + " id=f\"profile_demo_init_script\",\n", + " groups=[\"group-a\", \"group-b\"],\n", + " definition=ProfileDefinition(\n", + " display_name=\"Coder demo init script\",\n", + " description=\"This profile is used to demonstrate the use of an init script\",\n", + " slug=\"eoepca_demo_init_script\",\n", + " default=False,\n", + " kubespawner_override=KubespawnerOverride(\n", + " cpu_guarantee=1,\n", + " cpu_limit=2,\n", + " mem_guarantee=\"4G\",\n", + " mem_limit=\"6G\",\n", + " image=\"eoepca/pde-code-server:develop\",\n", + " ),\n", + " ),\n", + " node_selector=node_selector,\n", + " volumes=[calrissian_volume, workspace_volume],\n", + " config_maps=[init_cm],\n", + " pod_env_vars={\n", + " \"HOME\": \"/workspace\",\n", + " \"CONDA_ENVS_PATH\": \"/workspace/.envs\",\n", + " \"CONDARC\": \"/workspace/.condarc\",\n", + " \"XDG_RUNTIME_DIR\": \"/workspace/.local\",\n", + " \"CODE_SERVER_WS\": \"/workspace/mastering-app-package\",\n", + " },\n", + " init_containers=[init_container],\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## JupyterLab" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [], + "source": [ + "image = \"jupyter/scipy-notebook\"\n", + "\n", + "\n", + "eoepca_jupyter_lab_profile = Profile(\n", + " id=\"profile_jupyter_lab\",\n", + " groups=[\"group-c\"],\n", + " definition=ProfileDefinition(\n", + " display_name=\"Jupyter Lab\",\n", + " description=\"Jupyter Lab with Python 3.11\",\n", + " slug=\"eoepca_jupyter_lab\",\n", + " default=False,\n", + " kubespawner_override=KubespawnerOverride(\n", + " cpu_guarantee=1,\n", + " cpu_limit=2,\n", + " mem_guarantee=\"4G\",\n", + " mem_limit=\"6G\",\n", + " image=image,\n", + " ),\n", + " ),\n", + " node_selector=node_selector,\n", + " volumes=[workspace_volume],\n", + " config_maps=[],\n", + " pod_env_vars={\n", + " \"HOME\": \"/workspace\",\n", + " \"XDG_RUNTIME_DIR\": \"/workspace/.local\",\n", + " \"XDG_CONFIG_HOME\": \"/workspace/.config\",\n", + " },\n", + ")\n", + "\n", + "profiles.append(eoepca_jupyter_lab_profile)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Image pull secret\n" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [], + "source": [ + "image_pull_secret = ImagePullSecret(\n", + " name=\"cr-config\",\n", + " persist=False,\n", + " data=\"\",\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [], + "source": [ + "with open(\"manifests/manifest.yaml\", \"r\") as f:\n", + " content = yaml.safe_load_all(f.read())\n", + "\n", + "\n", + "\n", + "localstack_manifest = Manifest(\n", + " name=\"manifests\", key=\"manifests\", readonly=True, persist=False, content=[e for e in content]\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [], + "source": [ + "image = \"jupyter/scipy-notebook\"\n", + "\n", + "\n", + "eoepca_jupyter_lab_profile_2 = Profile(\n", + " id=\"profile_jupyter_lab_2\",\n", + " groups=[\"group-c\"],\n", + " definition=ProfileDefinition(\n", + " display_name=\"Jupyter Lab - profile 2\",\n", + " description=\"Jupyter Lab with Python 3.11 - demoes the use of an image pull secret\",\n", + " slug=\"eoepca_jupyter_lab_2\",\n", + " default=False,\n", + " kubespawner_override=KubespawnerOverride(\n", + " cpu_guarantee=1,\n", + " cpu_limit=2,\n", + " mem_guarantee=\"4G\",\n", + " mem_limit=\"6G\",\n", + " image=image,\n", + " ),\n", + " ),\n", + " node_selector=node_selector,\n", + " volumes=[workspace_volume],\n", + " config_maps=[],\n", + " pod_env_vars={\n", + " \"HOME\": \"/workspace\",\n", + " \"XDG_RUNTIME_DIR\": \"/workspace/.local\",\n", + " \"XDG_CONFIG_HOME\": \"/workspace/.config\",\n", + " },\n", + " image_pull_secrets=[image_pull_secret],\n", + " manifests=[localstack_manifest],\n", + ")\n", + "\n", + "profiles.append(eoepca_jupyter_lab_profile_2)" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Configuration" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [], + "source": [ + "config = Config(\n", + " profiles=profiles\n", + ")\n", + "\n", + "with open(\n", + " \"../files/hub/config.yml\", \"w\"\n", + ") as file:\n", + " yaml.dump(config.dict(), file)" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [], + "source": [ + "# profiles" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": ".env-config-generator", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.14" + }, + "orig_nbformat": 4 + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/config-generator/config-maps/bash-login b/config-generator/config-maps/bash-login new file mode 100644 index 0000000..743cb42 --- /dev/null +++ b/config-generator/config-maps/bash-login @@ -0,0 +1 @@ +source /workspace/.bashrc diff --git a/config-generator/config-maps/bash-rc b/config-generator/config-maps/bash-rc new file mode 100644 index 0000000..3f643a6 --- /dev/null +++ b/config-generator/config-maps/bash-rc @@ -0,0 +1,29 @@ +alias ll="ls -l" +alias calrissian="/opt/conda/bin/calrissian --pod-nodeselectors /etc/calrissian/pod-node-selector.yml --stdout /calrissian/results.json --max-ram 16G --max-cores "8" --tmp-outdir-prefix /calrissian/tmp/ --outdir /calrissian/" +alias cwltool="/opt/conda/bin/cwltool --podman" +. /home/jovyan/.bashrc + +#alias aws="aws --endpoint-url=http://localstack:4566" + +# >>> conda initialize >>> +# !! Contents within this block are managed by 'conda init' !! +__conda_setup="$('/opt/conda/bin/conda' 'shell.bash' 'hook' 2> /dev/null)" +if [ $? -eq 0 ]; then + eval "$__conda_setup" +else + if [ -f "/opt/conda/etc/profile.d/conda.sh" ]; then + . "/opt/conda/etc/profile.d/conda.sh" + else + export PATH="/srv/conda/bin:$PATH" + fi +fi +unset __conda_setup + +if [ -f "/opt/conda/etc/profile.d/mamba.sh" ]; then + . "/opt/conda/etc/profile.d/mamba.sh" +fi +# <<< conda initialize <<< + +a={{spawner.user.name}} + +alias aws="aws --endpoint-url=http://localstack-jupyter-{{spawner.user.name}}:4566" \ No newline at end of file diff --git a/config-generator/config-maps/conda-rc.yml b/config-generator/config-maps/conda-rc.yml new file mode 100644 index 0000000..f8a34eb --- /dev/null +++ b/config-generator/config-maps/conda-rc.yml @@ -0,0 +1,5 @@ +auto_update_conda: false +show_channel_urls: true +channels: + - conda-forge + - terradue diff --git a/config-generator/config-maps/init-qgis.sh b/config-generator/config-maps/init-qgis.sh new file mode 100644 index 0000000..738dc2e --- /dev/null +++ b/config-generator/config-maps/init-qgis.sh @@ -0,0 +1,17 @@ +mkdir -p /workspace/.config/autostart + + +cat < /workspace/.config/autostart/qgis.desktop +[Desktop Entry] +Encoding=UTF-8 +Version=0.9.4 +Type=Application +Name=qgis +Comment=qgis +Exec=qgis +OnlyShowIn=XFCE; +RunHook=0 +StartupNotify=false +Terminal=false +Hidden=false +EOF \ No newline at end of file diff --git a/config-generator/config-maps/init-stac.sh b/config-generator/config-maps/init-stac.sh new file mode 100644 index 0000000..f61d57a --- /dev/null +++ b/config-generator/config-maps/init-stac.sh @@ -0,0 +1,31 @@ +set -x + +cd /workspace + +git clone 'https://github.com/eoap/stac-eoap.git' + +code-server --install-extension ms-python.python +code-server --install-extension redhat.vscode-yaml +code-server --install-extension sbg-rabix.benten-cwl +code-server --install-extension ms-toolsai.jupyter + +ln -s /workspace/.local/share/code-server/extensions /workspace/extensions + +mkdir -p /workspace/User/ + +echo '{"workbench.colorTheme": "Visual Studio Dark"}' > /workspace/User/settings.json + +python -m venv /workspace/.venv +source /workspace/.venv/bin/activate +/workspace/.venv/bin/python -m pip install --no-cache-dir stactools rasterio requests stac-asset click-logging tabulate tqdm pystac-client ipykernel loguru scikit-image rio_stac boto3==1.35.23 + +/workspace/.venv/bin/python -m pip install --index-url https://test.pypi.org/simple cwl-wrapper + +/workspace/.venv/bin/python -m ipykernel install --user --name stac_env --display-name "Python (STAC)" + +export AWS_DEFAULT_REGION="us-east-1" +export AWS_ACCESS_KEY_ID="test" +export AWS_SECRET_ACCESS_KEY="test" +aws s3 mb s3://results --endpoint-url=http://localstack:4566 + +exit 0 \ No newline at end of file diff --git a/config-generator/config-maps/init.sh b/config-generator/config-maps/init.sh new file mode 100644 index 0000000..014fdf3 --- /dev/null +++ b/config-generator/config-maps/init.sh @@ -0,0 +1,14 @@ +set -x + +cd /workspace + +git clone 'https://github.com/eoap/mastering-app-package.git' + +code-server --install-extension ms-python.python +code-server --install-extension redhat.vscode-yaml +code-server --install-extension sbg-rabix.benten-cwl +code-server --install-extension ms-toolsai.jupyter + +ln -s /workspace/.local/share/code-server/extensions /workspace/extensions + +exit 0 diff --git a/config-generator/generate-config.sh b/config-generator/generate-config.sh new file mode 100755 index 0000000..279e056 --- /dev/null +++ b/config-generator/generate-config.sh @@ -0,0 +1,4 @@ +#exit 0 +export PYTHONPATH=$PWD/`dirname $0`/. + +.env-config-generator/bin/python3 -m generate_config \ No newline at end of file diff --git a/config-generator/generate_config.py b/config-generator/generate_config.py new file mode 100644 index 0000000..4847780 --- /dev/null +++ b/config-generator/generate_config.py @@ -0,0 +1,157 @@ +from models import * +import yaml +import os +from loguru import logger +from helpers import ( + load_config_map, + load_manifests, + create_init_container, + load_init_script, +) +import click + +storage_class_rwo = "standard" +storage_class_rwx = "standard" +profiles = [] +workspace_volume_size = "50Gi" +calrissian_volume_size = "50Gi" +image = "eoepca/pde-code-server:develop" +node_selector = {} + +# get the current directory +current_dir = os.path.dirname(os.path.realpath(__file__)) + +# load the manifests + +# localstack_manifest +localstack_manifest = load_manifests( + name="localstack", + key="localstack", + file_path=os.path.join(current_dir, "manifests/manifest.yaml"), +) + +# Dask Gateway manifest +dask_gateway_manifest = load_manifests( + name="dask-gateway", + key="dask-gateway", + file_path=os.path.join(current_dir, "manifests/dask-gateway.yaml"), +) + +kaniko_manifest = load_manifests( + name="kaniko", + key="kaniko", + file_path=os.path.join(current_dir, "manifests/kaniko.yaml"), +) +# volumes + +workspace_volume = Volume( + name="workspace-volume", + size=workspace_volume_size, + claim_name="workspace-claim", + mount_path="/workspace", + storage_class=storage_class_rwo, + access_modes=["ReadWriteOnce"], + volume_mount=VolumeMount(name="workspace-volume", mount_path="/workspace"), + persist=True, +) + +calrissian_volume = Volume( + name="calrissian-volume", + claim_name="calrissian-claim", + size=calrissian_volume_size, + storage_class=storage_class_rwx, + access_modes=["ReadWriteMany"], + volume_mount=VolumeMount(name="calrissian-volume", mount_path="/calrissian"), + persist=False, +) + + +bash_login_cm = load_config_map( + name="bash-login", + key="bash-login", + file_name=os.path.join(current_dir, "config-maps/bash-login"), + mount_path="/etc/profile.d/bash-login.sh", +) + +bash_rc_cm = load_config_map( + name="bash-rc", + key="bash-rc", + file_name=os.path.join(current_dir, "config-maps/bash-rc"), + mount_path="/workspace/.bashrc", +) + +init_cm = load_init_script(os.path.join(current_dir, "config-maps/init.sh")) + +init_container = create_init_container( + image=image, + volume=workspace_volume, + mount_path="/calrissian", +) + +profile_1 = Profile( + id=f"profile_1", + groups=["group-a", "group-b"], + definition=ProfileDefinition( + display_name="Coder demo init script", + description="This profile is used to demonstrate the use of an init script", + slug="profile_1", + default=False, + kubespawner_override=KubespawnerOverride( + cpu_guarantee=1, + cpu_limit=2, + mem_guarantee="4G", + mem_limit="6G", + image=image, + ), + ), + node_selector=node_selector, + volumes=[calrissian_volume, workspace_volume], + config_maps=[ + init_cm, + ConfigMap( + name="dask-gateway-config", + key="gateway", + mount_path="/etc/dask/gateway.yaml", + readonly=True, + ), + ConfigMap( + name="dask-gateway-config-ws", + key="gateway", + mount_path="/workspace/.config/dask/gateway.yaml", + readonly=True, + ), + bash_login_cm, + bash_rc_cm, + ], + pod_env_vars={ + "HOME": "/workspace", + "CONDA_ENVS_PATH": "/workspace/.envs", + "CONDARC": "/workspace/.condarc", + "XDG_RUNTIME_DIR": "/workspace/.local", + "CODE_SERVER_WS": "/workspace/mastering-app-package", + "DASK_GATEWAY": "http://traefik-dask-gw-jupyter-{{ spawner.user.name }}-dask-gateway.jupyter-{{ spawner.user.name }}.svc.cluster.local:80", + }, + init_containers=[init_container], + manifests=[localstack_manifest, dask_gateway_manifest, kaniko_manifest], + env_from_config_maps=["my-config"], + env_from_secrets=["my-secret", "data-by-name"], + secret_mounts=[ + SecretMount( + name="aws-credentials-{{ spawner.user.name }}", mount_path="/workspace/.aws" + ), + SecretMount(name="data-by-name", mount_path="/workspace/.data-by-name"), + SecretMount( + name="eoepca-plus-secret-ro", + mount_path="/workspace/.docker/config.json", + sub_path=".dockerconfigjson", + ), + ], + image_pull_secrets=[ImagePullSecret(name="eoepca-plus-secret-ro")], +) + +profiles.append(profile_1) + +config = Config(profiles=profiles) + +with open("files/hub/config.yml", "w") as file: + yaml.dump(config.dict(), file, width=200) diff --git a/config-generator/helpers.py b/config-generator/helpers.py new file mode 100644 index 0000000..1ae9d98 --- /dev/null +++ b/config-generator/helpers.py @@ -0,0 +1,62 @@ +import yaml + +from models import ( + Volume, + InitContainer, + VolumeMount, + ConfigMap, + InitContainerVolumeMount, + Manifest, +) + + +def load_config_map(name, key, file_name, mount_path): + with open(file_name, "r") as f: + content = f.read() + return ConfigMap( + name=name, + key=key, + content=content, + readonly=True, + persist=True, + mount_path=mount_path, + ) + + +def load_manifests(name, key, file_path): + with open(file_path, "r") as f: + content = yaml.safe_load_all(f.read()) + return Manifest( + name=name, key=key, readonly=True, persist=False, content=[e for e in content] + ) + + +def create_init_container(image: str, volume: Volume, mount_path: str) -> InitContainer: + + init_context_volume_mount = InitContainerVolumeMount( + mount_path="/opt/init/.init.sh", name="init", sub_path="init" + ) + + return InitContainer( + name="init-file-on-volume", + image=image, + command=["sh", "-c", "sh /opt/init/.init.sh"], + volume_mounts=[ + VolumeMount(name=volume.name, mount_path=mount_path), + init_context_volume_mount, + ], + ) + + +def load_init_script(file_name: str) -> ConfigMap: + with open(file_name, "r") as f: + content = f.read() + return ConfigMap( + name="init", + key="init", + content=content, + readonly=True, + persist=False, + mount_path="/opt/init/.init.sh", + default_mode="0660", + ) diff --git a/config-generator/manifests/dask-gateway.yaml b/config-generator/manifests/dask-gateway.yaml new file mode 100644 index 0000000..c760862 --- /dev/null +++ b/config-generator/manifests/dask-gateway.yaml @@ -0,0 +1,109 @@ +apiVersion: helm.crossplane.io/v1beta1 +kind: Release +metadata: + name: dask-gw-jupyter-{{ spawner.user.name }} + namespace: jupyter-{{ spawner.user.name }} +spec: + forProvider: + chart: + name: dask-gateway + version: "2024.1.0" + repository: https://helm.dask.org + namespace: jupyter-{{ spawner.user.name }} + values: + gateway: + backend: + image: + name: ghcr.io/fabricebrito/dev-platform-dask-gateway/worker + tag: 1.0.0 + extraConfig: + dask_gateway_config.py: | + c = get_config() + from dask_gateway_server.options import Options, String, Integer, Float + c.Backend.cluster_options = Options( + Float("worker_cores_limit", default=1, label="Worker Cores Limit"), + Float("worker_cores", default=1, label="Worker Cores"), + String("worker_memory", default="1 G", label="Worker Memory"), + String("image", default="daskgateway/dask-worker:latest", label="Worker Image") + ) + traefik: + service: + type: "ClusterIP" + providerConfigRef: + name: helm-provider +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: dask-gateway-config +data: + gateway: | + gateway: + address: http://traefik-dask-gw-jupyter-{{ spawner.user.name }}-dask-gateway.jupyter-{{ spawner.user.name }}.svc.cluster.local:80 + + cluster: + options: + image: "ghcr.io/fabricebrito/dev-platform-dask-gateway/worker:1.0.0" + worker_cores: 0.5 + worker_cores_limit: 1 + worker_memory: "4 G" +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: dask-gateway-config-ws +data: + gateway: | + gateway: + address: http://traefik-dask-gw-jupyter-{{ spawner.user.name }}-dask-gateway.jupyter-{{ spawner.user.name }}.svc.cluster.local:80 + + cluster: + options: + image: "ghcr.io/fabricebrito/dev-platform-dask-gateway/worker:1.0.0" + worker_cores: 0.5 + worker_cores_limit: 1 + worker_memory: "4 G" +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: access-services + namespace: jupyter-{{ spawner.user.name }} +rules: + - verbs: + - get + - list + - watch + apiGroups: + - '' + resources: + - services + - pods/exec + - verbs: + - get + - list + - watch + apiGroups: + - '' + resources: + - pods + - verbs: + - create + apiGroups: + - '' + resources: + - pods/portforward +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: bind-default-to-services + namespace: jupyter-{{ spawner.user.name }} +subjects: + - kind: ServiceAccount + name: default + namespace: jupyter-{{ spawner.user.name }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: access-services \ No newline at end of file diff --git a/config-generator/manifests/kaniko.yaml b/config-generator/manifests/kaniko.yaml new file mode 100644 index 0000000..e7280b9 --- /dev/null +++ b/config-generator/manifests/kaniko.yaml @@ -0,0 +1,58 @@ + apiVersion: v1 + kind: Pod + metadata: + name: kaniko-build + namespace: jupyter-{{ spawner.user.name }} + spec: + containers: + - name: kaniko + image: gcr.io/kaniko-project/executor:debug + command: + - /bin/sh + - -c + - "sleep infinity" # Keep the container running for manual access + volumeMounts: + - name: build-context + mountPath: /calrissian # Mount your build files here + - name: kaniko-secret + mountPath: /kaniko/.docker + restartPolicy: Never + volumes: + - name: build-context + persistentVolumeClaim: + claimName: calrissian-claim # Replace with your PVC name + - name: kaniko-secret + secret: + secretName: kaniko-secret + items: + - key: .dockerconfigjson + path: config.json +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: pod-exec + namespace: jupyter-{{ spawner.user.name }} +rules: + - verbs: + - get + - list + - watch + apiGroups: + - '' + resources: + - pods/exec +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: bind-default-to-opd-exec + namespace: jupyter-{{ spawner.user.name }} +subjects: + - kind: ServiceAccount + name: default + namespace: jupyter-{{ spawner.user.name }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: pod-exec diff --git a/config-generator/manifests/manifest.yaml b/config-generator/manifests/manifest.yaml new file mode 100644 index 0000000..829b1f7 --- /dev/null +++ b/config-generator/manifests/manifest.yaml @@ -0,0 +1,505 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: localstack +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: localstack +rules: +- apiGroups: [""] # "" indicates the core API group + resources: ["pods"] + verbs: ["*"] +- apiGroups: [""] + resources: ["pods/log"] + verbs: ["get"] +- apiGroups: [""] + resources: ["pods/exec"] + verbs: ["get", "create"] +- apiGroups: [""] + resources: ["services"] + verbs: ["get", "list"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: localstack +subjects: +- kind: ServiceAccount + name: localstack +roleRef: + kind: Role + name: localstack + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: v1 +kind: Service +metadata: + name: localstack +spec: + type: ClusterIP + ports: + - name: edge + port: 4566 + targetPort: 4566 + - name: "external-service-port-4510" + port: 4510 + targetPort: "ext-svc-4510" + - name: "external-service-port-4511" + port: 4511 + targetPort: "ext-svc-4511" + - name: "external-service-port-4512" + port: 4512 + targetPort: "ext-svc-4512" + - name: "external-service-port-4513" + port: 4513 + targetPort: "ext-svc-4513" + - name: "external-service-port-4514" + port: 4514 + targetPort: "ext-svc-4514" + - name: "external-service-port-4515" + port: 4515 + targetPort: "ext-svc-4515" + - name: "external-service-port-4516" + port: 4516 + targetPort: "ext-svc-4516" + - name: "external-service-port-4517" + port: 4517 + targetPort: "ext-svc-4517" + - name: "external-service-port-4518" + port: 4518 + targetPort: "ext-svc-4518" + - name: "external-service-port-4519" + port: 4519 + targetPort: "ext-svc-4519" + - name: "external-service-port-4520" + port: 4520 + targetPort: "ext-svc-4520" + - name: "external-service-port-4521" + port: 4521 + targetPort: "ext-svc-4521" + - name: "external-service-port-4522" + port: 4522 + targetPort: "ext-svc-4522" + - name: "external-service-port-4523" + port: 4523 + targetPort: "ext-svc-4523" + - name: "external-service-port-4524" + port: 4524 + targetPort: "ext-svc-4524" + - name: "external-service-port-4525" + port: 4525 + targetPort: "ext-svc-4525" + - name: "external-service-port-4526" + port: 4526 + targetPort: "ext-svc-4526" + - name: "external-service-port-4527" + port: 4527 + targetPort: "ext-svc-4527" + - name: "external-service-port-4528" + port: 4528 + targetPort: "ext-svc-4528" + - name: "external-service-port-4529" + port: 4529 + targetPort: "ext-svc-4529" + - name: "external-service-port-4530" + port: 4530 + targetPort: "ext-svc-4530" + - name: "external-service-port-4531" + port: 4531 + targetPort: "ext-svc-4531" + - name: "external-service-port-4532" + port: 4532 + targetPort: "ext-svc-4532" + - name: "external-service-port-4533" + port: 4533 + targetPort: "ext-svc-4533" + - name: "external-service-port-4534" + port: 4534 + targetPort: "ext-svc-4534" + - name: "external-service-port-4535" + port: 4535 + targetPort: "ext-svc-4535" + - name: "external-service-port-4536" + port: 4536 + targetPort: "ext-svc-4536" + - name: "external-service-port-4537" + port: 4537 + targetPort: "ext-svc-4537" + - name: "external-service-port-4538" + port: 4538 + targetPort: "ext-svc-4538" + - name: "external-service-port-4539" + port: 4539 + targetPort: "ext-svc-4539" + - name: "external-service-port-4540" + port: 4540 + targetPort: "ext-svc-4540" + - name: "external-service-port-4541" + port: 4541 + targetPort: "ext-svc-4541" + - name: "external-service-port-4542" + port: 4542 + targetPort: "ext-svc-4542" + - name: "external-service-port-4543" + port: 4543 + targetPort: "ext-svc-4543" + - name: "external-service-port-4544" + port: 4544 + targetPort: "ext-svc-4544" + - name: "external-service-port-4545" + port: 4545 + targetPort: "ext-svc-4545" + - name: "external-service-port-4546" + port: 4546 + targetPort: "ext-svc-4546" + - name: "external-service-port-4547" + port: 4547 + targetPort: "ext-svc-4547" + - name: "external-service-port-4548" + port: 4548 + targetPort: "ext-svc-4548" + - name: "external-service-port-4549" + port: 4549 + targetPort: "ext-svc-4549" + - name: "external-service-port-4550" + port: 4550 + targetPort: "ext-svc-4550" + - name: "external-service-port-4551" + port: 4551 + targetPort: "ext-svc-4551" + - name: "external-service-port-4552" + port: 4552 + targetPort: "ext-svc-4552" + - name: "external-service-port-4553" + port: 4553 + targetPort: "ext-svc-4553" + - name: "external-service-port-4554" + port: 4554 + targetPort: "ext-svc-4554" + - name: "external-service-port-4555" + port: 4555 + targetPort: "ext-svc-4555" + - name: "external-service-port-4556" + port: 4556 + targetPort: "ext-svc-4556" + - name: "external-service-port-4557" + port: 4557 + targetPort: "ext-svc-4557" + - name: "external-service-port-4558" + port: 4558 + targetPort: "ext-svc-4558" + - name: "external-service-port-4559" + port: 4559 + targetPort: "ext-svc-4559" + selector: + app.kubernetes.io/name: localstack + app.kubernetes.io/instance: localstack +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: localstack +spec: + replicas: 1 + strategy: + type: RollingUpdate + selector: + matchLabels: + app.kubernetes.io/name: localstack + app.kubernetes.io/instance: localstack + template: + metadata: + labels: + app: localstack-{{ spawner.user.name }} + app.kubernetes.io/name: localstack + app.kubernetes.io/instance: localstack + spec: + serviceAccountName: localstack + securityContext: + {} + containers: + - name: localstack + securityContext: + {} + image: "localstack/localstack:latest" + imagePullPolicy: IfNotPresent + ports: + - name: edge + containerPort: 4566 + protocol: TCP + - name: "ext-svc-4510" + containerPort: 4510 + protocol: TCP + - name: "ext-svc-4511" + containerPort: 4511 + protocol: TCP + - name: "ext-svc-4512" + containerPort: 4512 + protocol: TCP + - name: "ext-svc-4513" + containerPort: 4513 + protocol: TCP + - name: "ext-svc-4514" + containerPort: 4514 + protocol: TCP + - name: "ext-svc-4515" + containerPort: 4515 + protocol: TCP + - name: "ext-svc-4516" + containerPort: 4516 + protocol: TCP + - name: "ext-svc-4517" + containerPort: 4517 + protocol: TCP + - name: "ext-svc-4518" + containerPort: 4518 + protocol: TCP + - name: "ext-svc-4519" + containerPort: 4519 + protocol: TCP + - name: "ext-svc-4520" + containerPort: 4520 + protocol: TCP + - name: "ext-svc-4521" + containerPort: 4521 + protocol: TCP + - name: "ext-svc-4522" + containerPort: 4522 + protocol: TCP + - name: "ext-svc-4523" + containerPort: 4523 + protocol: TCP + - name: "ext-svc-4524" + containerPort: 4524 + protocol: TCP + - name: "ext-svc-4525" + containerPort: 4525 + protocol: TCP + - name: "ext-svc-4526" + containerPort: 4526 + protocol: TCP + - name: "ext-svc-4527" + containerPort: 4527 + protocol: TCP + - name: "ext-svc-4528" + containerPort: 4528 + protocol: TCP + - name: "ext-svc-4529" + containerPort: 4529 + protocol: TCP + - name: "ext-svc-4530" + containerPort: 4530 + protocol: TCP + - name: "ext-svc-4531" + containerPort: 4531 + protocol: TCP + - name: "ext-svc-4532" + containerPort: 4532 + protocol: TCP + - name: "ext-svc-4533" + containerPort: 4533 + protocol: TCP + - name: "ext-svc-4534" + containerPort: 4534 + protocol: TCP + - name: "ext-svc-4535" + containerPort: 4535 + protocol: TCP + - name: "ext-svc-4536" + containerPort: 4536 + protocol: TCP + - name: "ext-svc-4537" + containerPort: 4537 + protocol: TCP + - name: "ext-svc-4538" + containerPort: 4538 + protocol: TCP + - name: "ext-svc-4539" + containerPort: 4539 + protocol: TCP + - name: "ext-svc-4540" + containerPort: 4540 + protocol: TCP + - name: "ext-svc-4541" + containerPort: 4541 + protocol: TCP + - name: "ext-svc-4542" + containerPort: 4542 + protocol: TCP + - name: "ext-svc-4543" + containerPort: 4543 + protocol: TCP + - name: "ext-svc-4544" + containerPort: 4544 + protocol: TCP + - name: "ext-svc-4545" + containerPort: 4545 + protocol: TCP + - name: "ext-svc-4546" + containerPort: 4546 + protocol: TCP + - name: "ext-svc-4547" + containerPort: 4547 + protocol: TCP + - name: "ext-svc-4548" + containerPort: 4548 + protocol: TCP + - name: "ext-svc-4549" + containerPort: 4549 + protocol: TCP + - name: "ext-svc-4550" + containerPort: 4550 + protocol: TCP + - name: "ext-svc-4551" + containerPort: 4551 + protocol: TCP + - name: "ext-svc-4552" + containerPort: 4552 + protocol: TCP + - name: "ext-svc-4553" + containerPort: 4553 + protocol: TCP + - name: "ext-svc-4554" + containerPort: 4554 + protocol: TCP + - name: "ext-svc-4555" + containerPort: 4555 + protocol: TCP + - name: "ext-svc-4556" + containerPort: 4556 + protocol: TCP + - name: "ext-svc-4557" + containerPort: 4557 + protocol: TCP + - name: "ext-svc-4558" + containerPort: 4558 + protocol: TCP + - name: "ext-svc-4559" + containerPort: 4559 + protocol: TCP + livenessProbe: + failureThreshold: 3 + initialDelaySeconds: 0 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + httpGet: + path: /_localstack/health + port: edge + readinessProbe: + failureThreshold: 3 + initialDelaySeconds: 0 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + httpGet: + path: /_localstack/health + port: edge + resources: {} + env: + - name: DEBUG + value: "0" + - name: EXTERNAL_SERVICE_PORTS_START + value: "4510" + - name: EXTERNAL_SERVICE_PORTS_END + value: "4560" + - name: LOCALSTACK_K8S_SERVICE_NAME + value: localstack + - name: LOCALSTACK_K8S_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: LAMBDA_RUNTIME_EXECUTOR + value: "docker" + - name: LAMBDA_K8S_IMAGE_PREFIX + value: "localstack/lambda-" + - name: LAMBDA_RUNTIME_ENVIRONMENT_TIMEOUT + value: "60" + - name: OVERRIDE_IN_DOCKER + value: "1" + volumes: [] +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: my-config +data: + ENV_VAR1: value1 + ENV_VAR2: value2 +--- +apiVersion: v1 +kind: Secret +metadata: + name: my-secret +type: Opaque +data: + SECRET_KEY1: dmFsdWUx # base64 for "value1" + SECRET_KEY2: dmFsdWUy # base64 for "value2" +--- +apiVersion: v1 +kind: Secret +metadata: + name: aws-credentials-{{ spawner.user.name }} +type: Opaque +data: + credentials: W2RlZmF1bHRdCmF3c19hY2Nlc3Nfa2V5X2lkPUFTSUFJT1NGT0ROTjdFWEFNUExFCmF3c19zZWNyZXRfYWNjZXNzX2tleT13SmFsclhVdG5GRU1JL0s3TURFTkcvYlB4UmZpQ1lFWEFNUExFS0VZCmF3c19zZXNzaW9uX3Rva2VuPUlRb0piM2pySmdCV05FTE5Hb2xHSkxFT3RTVEFOR1k0TFlPNUk0SzVOUlZFS1pPTkNTTk1HRlNUS1FNSUxXUjJPUzAwRklDRTExSlg= +# --- +# apiVersion: helm.crossplane.io/v1beta1 +# kind: Release +# metadata: +# name: wordpress-jupyter-{{ spawner.user.name }} +# namespace: jupyter-{{ spawner.user.name }} +# spec: +# forProvider: +# chart: +# name: wordpress +# version: "24.1.4" +# repository: oci://registry-1.docker.io/bitnamicharts/ +# namespace: jupyter-{{ spawner.user.name }} +# values: +# service: +# type: ClusterIP +# providerConfigRef: +# name: helm-provider +--- +apiVersion: external-secrets.io/v1beta1 +kind: ExternalSecret +metadata: + name: data-by-name + namespace: jupyter-{{ spawner.user.name }} +spec: + refreshInterval: 15s + secretStoreRef: + kind: ClusterSecretStore + name: k8s-secret-store + target: + name: data-by-name + creationPolicy: Owner + + data: + - secretKey: secret-value + remoteRef: + key: secret-one # name of the secret + property: "the-key" # key in the secret +--- +apiVersion: external-secrets.io/v1beta1 +kind: ExternalSecret +metadata: + name: eoepca-plus-secret-ro + namespace: jupyter-{{ spawner.user.name }} +spec: + refreshInterval: 15s + secretStoreRef: + kind: ClusterSecretStore + name: k8s-secret-store + target: + name: eoepca-plus-secret-ro + creationPolicy: Owner + + data: + - secretKey: .dockerconfigjson + remoteRef: + key: eoepca-plus-secret-ro # name of the secret + property: ".dockerconfigjson" # key in the secret diff --git a/config-generator/models.py b/config-generator/models.py new file mode 100644 index 0000000..4715396 --- /dev/null +++ b/config-generator/models.py @@ -0,0 +1,157 @@ +from enum import Enum +from typing import Dict, List, Optional, Union + +from pydantic import BaseModel + + +class ConfigMapKeyRef(BaseModel): + name: str + key: str + + +class ConfigMapEnvVarReference(BaseModel): + from_config_map: ConfigMapKeyRef + + +class SubjectKind(str, Enum): + service_account = "ServiceAccount" + user = "User" + + +class Verb(str, Enum): + get = "get" + list = "list" + watch = "watch" + create = "create" + update = "update" + patch = "patch" + delete = "delete" + deletecollection = "deletecollection" + + +class Subject(BaseModel): + name: str + kind: SubjectKind + + +class Role(BaseModel): + name: str + resources: List[str] + verbs: List[Verb] + api_groups: Optional[List[str]] = [""] + + +class RoleBinding(BaseModel): + name: str + subjects: List[Subject] + role: Role + persist: bool = True + + +class VolumeMount(BaseModel): + """volume mount object""" + + name: str + mount_path: str + + +class InitContainerVolumeMount(VolumeMount): + sub_path: str + + +class Volume(BaseModel): + """volume object""" + + name: str + claim_name: str + size: str + storage_class: str + access_modes: List[str] + volume_mount: VolumeMount + persist: bool + + +class Manifest(BaseModel): + name: str + key: str + content: Optional[List[Dict]] = None + persist: Optional[bool] = True + + +class ConfigMap(BaseModel): + """config map object""" + + name: str + key: str + mount_path: Optional[str] = None + default_mode: Optional[str] = None + readonly: bool + content: Optional[str] = None + persist: Optional[bool] = True + + +class KubespawnerOverride(BaseModel): + """kubespawner override object""" + + cpu_limit: int + cpu_guarantee: Optional[int] = None + mem_limit: str + mem_guarantee: Optional[str] = None + image: str + extra_resource_limits: Optional[dict] = {} + extra_resource_guarantees: Optional[dict] = {} + + +class InitContainer(BaseModel): + name: str + image: str + command: List[str] + volume_mounts: list[VolumeMount | InitContainerVolumeMount] + + +class ProfileDefinition(BaseModel): + """profile definition object""" + + display_name: str + description: Optional[str] = None + slug: str + default: bool + kubespawner_override: KubespawnerOverride + + +class ImagePullSecret(BaseModel): + name: str + persist: bool = True + data: Optional[str] = None + + +class SecretMount(BaseModel): + name: str + mount_path: str + sub_path: Optional[str] = None + + +class Profile(BaseModel): + """profile object""" + + id: str + groups: List[str] + definition: ProfileDefinition + config_maps: Optional[List[ConfigMap]] = None + volumes: Optional[List[Volume]] = None + pod_env_vars: Optional[Dict[str, Union[str, ConfigMapEnvVarReference]]] = None + default_url: Optional[str] = None + node_selector: dict + role_bindings: Optional[List[RoleBinding]] = None + image_pull_secrets: Optional[List[ImagePullSecret]] = [] + init_containers: Optional[List[InitContainer]] = [] + manifests: Optional[List[Manifest]] = None + env_from_config_maps: Optional[List[str]] = None + env_from_secrets: Optional[List[str]] = None + secret_mounts: Optional[List[SecretMount]] = None + + +class Config(BaseModel): + """config object""" + + profiles: List[Profile] diff --git a/config-generator/requirements.txt b/config-generator/requirements.txt new file mode 100644 index 0000000..3c47a85 --- /dev/null +++ b/config-generator/requirements.txt @@ -0,0 +1,3 @@ +ipykernel +pydantic +pyyaml \ No newline at end of file diff --git a/eoepca-demo/config.yml b/eoepca-demo/config.yml new file mode 100644 index 0000000..61b05c9 --- /dev/null +++ b/eoepca-demo/config.yml @@ -0,0 +1,942 @@ +profiles: +- config_maps: + - content: "alias ll=\"ls -l\"\nalias calrissian=\"/opt/conda/bin/calrissian --pod-nodeselectors\ + \ /etc/calrissian/pod-node-selector.yml --stdout /calrissian/results.json --max-ram\ + \ 16G --max-cores \"8\" --tmp-outdir-prefix /calrissian/tmp/ --outdir /calrissian/\"\ + \nalias cwltool=\"/opt/conda/bin/cwltool --podman\"\n. /home/jovyan/.bashrc\n\ + \nalias aws=\"aws --endpoint-url=http://localstack:4566\"\n\n# >>> conda initialize\ + \ >>>\n# !! Contents within this block are managed by 'conda init' !!\n__conda_setup=\"\ + $('/opt/conda/bin/conda' 'shell.bash' 'hook' 2> /dev/null)\"\nif [ $? -eq 0\ + \ ]; then\n eval \"$__conda_setup\"\nelse\n if [ -f \"/opt/conda/etc/profile.d/conda.sh\"\ + \ ]; then\n . \"/opt/conda/etc/profile.d/conda.sh\"\n else\n \ + \ export PATH=\"/srv/conda/bin:$PATH\"\n fi\nfi\nunset __conda_setup\n\n\ + if [ -f \"/opt/conda/etc/profile.d/mamba.sh\" ]; then\n . \"/opt/conda/etc/profile.d/mamba.sh\"\ + \nfi\n# <<< conda initialize <<<\n" + default_mode: null + key: bash-rc + mount_path: /workspace/.bashrc + name: bash-rc + persist: false + readonly: true + default_url: null + definition: + default: false + description: null + display_name: Code Server Small + kubespawner_override: + cpu_guarantee: null + cpu_limit: 2 + extra_resource_guarantees: {} + extra_resource_limits: {} + image: eoepca/pde-code-server:develop + mem_guarantee: null + mem_limit: 8G + slug: ellip_studio_coder_slug_s + groups: + - group-a + - group-b + id: profile_studio_coder1 + image_pull_secrets: [] + init_containers: [] + manifests: null + node_selector: {} + pod_env_vars: + CONDA_ENVS_PATH: /workspace/.envs + HOME: /workspace + role_bindings: null + volumes: + - access_modes: + - ReadWriteMany + claim_name: calrissian-claim + name: calrissian-volume + persist: false + size: 50Gi + storage_class: managed-nfs-storage + volume_mount: + mount_path: /calrissian + name: calrissian-volume + - access_modes: + - ReadWriteOnce + claim_name: workspace-claim + name: workspace-volume + persist: false + size: 50Gi + storage_class: managed-nfs-storage + volume_mount: + mount_path: /workspace + name: workspace-volume +- config_maps: + - content: "alias ll=\"ls -l\"\nalias calrissian=\"/opt/conda/bin/calrissian --pod-nodeselectors\ + \ /etc/calrissian/pod-node-selector.yml --stdout /calrissian/results.json --max-ram\ + \ 16G --max-cores \"8\" --tmp-outdir-prefix /calrissian/tmp/ --outdir /calrissian/\"\ + \nalias cwltool=\"/opt/conda/bin/cwltool --podman\"\n. /home/jovyan/.bashrc\n\ + \nalias aws=\"aws --endpoint-url=http://localstack:4566\"\n\n# >>> conda initialize\ + \ >>>\n# !! Contents within this block are managed by 'conda init' !!\n__conda_setup=\"\ + $('/opt/conda/bin/conda' 'shell.bash' 'hook' 2> /dev/null)\"\nif [ $? -eq 0\ + \ ]; then\n eval \"$__conda_setup\"\nelse\n if [ -f \"/opt/conda/etc/profile.d/conda.sh\"\ + \ ]; then\n . \"/opt/conda/etc/profile.d/conda.sh\"\n else\n \ + \ export PATH=\"/srv/conda/bin:$PATH\"\n fi\nfi\nunset __conda_setup\n\n\ + if [ -f \"/opt/conda/etc/profile.d/mamba.sh\" ]; then\n . \"/opt/conda/etc/profile.d/mamba.sh\"\ + \nfi\n# <<< conda initialize <<<\n" + default_mode: null + key: bash-rc + mount_path: /workspace/.bashrc + name: bash-rc + persist: false + readonly: true + default_url: null + definition: + default: false + description: null + display_name: Code Server Medium + kubespawner_override: + cpu_guarantee: null + cpu_limit: 4 + extra_resource_guarantees: {} + extra_resource_limits: {} + image: eoepca/pde-code-server:develop + mem_guarantee: null + mem_limit: 12G + slug: ellip_studio_coder_slug_m + groups: + - group-a + - group-b + id: profile_studio_coder2 + image_pull_secrets: [] + init_containers: [] + manifests: null + node_selector: {} + pod_env_vars: + CONDA_ENVS_PATH: /workspace/.envs + HOME: /workspace + role_bindings: null + volumes: + - access_modes: + - ReadWriteMany + claim_name: calrissian-claim + name: calrissian-volume + persist: false + size: 50Gi + storage_class: managed-nfs-storage + volume_mount: + mount_path: /calrissian + name: calrissian-volume + - access_modes: + - ReadWriteOnce + claim_name: workspace-claim + name: workspace-volume + persist: false + size: 50Gi + storage_class: managed-nfs-storage + volume_mount: + mount_path: /workspace + name: workspace-volume +- config_maps: + - content: 'set -x + + + cd /workspace + + + git clone ''https://github.com/eoap/mastering-app-package.git'' + + + code-server --install-extension ms-python.python + + code-server --install-extension redhat.vscode-yaml + + code-server --install-extension sbg-rabix.benten-cwl + + code-server --install-extension ms-toolsai.jupyter + + + ln -s /workspace/.local/share/code-server/extensions /workspace/extensions + + + exit 0 + + ' + default_mode: '0660' + key: init + mount_path: /opt/init/.init.sh + name: init + persist: false + readonly: true + default_url: null + definition: + default: false + description: This profile is used to demonstrate the use of an init script + display_name: Coder demo init script + kubespawner_override: + cpu_guarantee: 1 + cpu_limit: 2 + extra_resource_guarantees: {} + extra_resource_limits: {} + image: eoepca/pde-code-server:develop + mem_guarantee: 4G + mem_limit: 6G + slug: eoepca_demo_init_script + groups: + - group-a + - group-b + id: profile_demo_init_script + image_pull_secrets: [] + init_containers: + - command: + - sh + - -c + - sh /opt/init/.init.sh + image: eoepca/pde-code-server:develop + name: init-file-on-volume + volume_mounts: + - mount_path: /workspace + name: workspace-volume + - mount_path: /opt/init/.init.sh + name: init + sub_path: init + manifests: null + node_selector: {} + pod_env_vars: + CODE_SERVER_WS: /workspace/mastering-app-package + CONDARC: /workspace/.condarc + CONDA_ENVS_PATH: /workspace/.envs + HOME: /workspace + XDG_RUNTIME_DIR: /workspace/.local + role_bindings: null + volumes: + - access_modes: + - ReadWriteMany + claim_name: calrissian-claim + name: calrissian-volume + persist: false + size: 50Gi + storage_class: managed-nfs-storage + volume_mount: + mount_path: /calrissian + name: calrissian-volume + - access_modes: + - ReadWriteOnce + claim_name: workspace-claim + name: workspace-volume + persist: false + size: 50Gi + storage_class: managed-nfs-storage + volume_mount: + mount_path: /workspace + name: workspace-volume +- config_maps: [] + default_url: null + definition: + default: false + description: Jupyter Lab with Python 3.11 + display_name: Jupyter Lab + kubespawner_override: + cpu_guarantee: 1 + cpu_limit: 2 + extra_resource_guarantees: {} + extra_resource_limits: {} + image: jupyter/scipy-notebook + mem_guarantee: 4G + mem_limit: 6G + slug: eoepca_jupyter_lab + groups: + - group-c + id: profile_jupyter_lab + image_pull_secrets: [] + init_containers: [] + manifests: null + node_selector: {} + pod_env_vars: + HOME: /workspace + XDG_CONFIG_HOME: /workspace/.config + XDG_RUNTIME_DIR: /workspace/.local + role_bindings: null + volumes: + - access_modes: + - ReadWriteOnce + claim_name: workspace-claim + name: workspace-volume + persist: false + size: 50Gi + storage_class: managed-nfs-storage + volume_mount: + mount_path: /workspace + name: workspace-volume +- config_maps: [] + default_url: null + definition: + default: false + description: Jupyter Lab with Python 3.11 private image - demoes the use of an + image pull secret + display_name: Jupyter Lab - profile 2 + kubespawner_override: + cpu_guarantee: 1 + cpu_limit: 2 + extra_resource_guarantees: {} + extra_resource_limits: {} + image: eoepca/iat-jupyterlab:develop + mem_guarantee: 4G + mem_limit: 6G + slug: eoepca_jupyter_lab_2 + groups: + - group-c + id: profile_jupyter_lab_2 + image_pull_secrets: + - data: null + - data: '' + name: cr-config + persist: false + init_containers: [] + manifests: null + node_selector: {} + pod_env_vars: + HOME: /workspace + XDG_CONFIG_HOME: /workspace/.config + XDG_RUNTIME_DIR: /workspace/.local + role_bindings: null + volumes: + - access_modes: + - ReadWriteOnce + claim_name: workspace-claim + name: workspace-volume + persist: false + size: 50Gi + storage_class: managed-nfs-storage + volume_mount: + mount_path: /workspace + name: workspace-volume +- config_maps: + - content: "set -x \n\ncd /workspace\n\ngit clone 'https://github.com/eoap/stac-eoap.git'\n\ + \ncode-server --install-extension ms-python.python \ncode-server --install-extension\ + \ redhat.vscode-yaml\ncode-server --install-extension sbg-rabix.benten-cwl\n\ + code-server --install-extension ms-toolsai.jupyter\n\nln -s /workspace/.local/share/code-server/extensions\ + \ /workspace/extensions\n\nmkdir -p /workspace/User/\n\necho '{\"workbench.colorTheme\"\ + : \"Visual Studio Dark\"}' > /workspace/User/settings.json\n\npython -m venv\ + \ /workspace/.venv\nsource /workspace/.venv/bin/activate\n/workspace/.venv/bin/python\ + \ -m pip install --no-cache-dir stactools rasterio requests stac-asset click-logging\ + \ tabulate tqdm pystac-client ipykernel loguru scikit-image rio_stac boto3==1.35.23\n\ + \n/workspace/.venv/bin/python -m pip install --index-url https://test.pypi.org/simple\ + \ cwl-wrapper\n\n/workspace/.venv/bin/python -m ipykernel install --user --name\ + \ stac_env --display-name \"Python (STAC)\"\n\nexport AWS_DEFAULT_REGION=\"\ + us-east-1\"\nexport AWS_ACCESS_KEY_ID=\"test\"\nexport AWS_SECRET_ACCESS_KEY=\"\ + test\"\naws s3 mb s3://results --endpoint-url=http://localstack:4566\n\nexit\ + \ 0" + default_mode: null + key: init + mount_path: /opt/init/.init.sh + name: init + persist: false + readonly: true + - content: "alias ll=\"ls -l\"\nalias calrissian=\"/opt/conda/bin/calrissian --pod-nodeselectors\ + \ /etc/calrissian/pod-node-selector.yml --stdout /calrissian/results.json --max-ram\ + \ 16G --max-cores \"8\" --tmp-outdir-prefix /calrissian/tmp/ --outdir /calrissian/\"\ + \nalias cwltool=\"/opt/conda/bin/cwltool --podman\"\n. /home/jovyan/.bashrc\n\ + \nalias aws=\"aws --endpoint-url=http://localstack:4566\"\n\n# >>> conda initialize\ + \ >>>\n# !! Contents within this block are managed by 'conda init' !!\n__conda_setup=\"\ + $('/opt/conda/bin/conda' 'shell.bash' 'hook' 2> /dev/null)\"\nif [ $? -eq 0\ + \ ]; then\n eval \"$__conda_setup\"\nelse\n if [ -f \"/opt/conda/etc/profile.d/conda.sh\"\ + \ ]; then\n . \"/opt/conda/etc/profile.d/conda.sh\"\n else\n \ + \ export PATH=\"/srv/conda/bin:$PATH\"\n fi\nfi\nunset __conda_setup\n\n\ + if [ -f \"/opt/conda/etc/profile.d/mamba.sh\" ]; then\n . \"/opt/conda/etc/profile.d/mamba.sh\"\ + \nfi\n# <<< conda initialize <<<\n" + default_mode: null + key: bash-rc + mount_path: /workspace/.bashrc + name: bash-rc + persist: false + readonly: true + - content: 'source /workspace/.bashrc + + ' + default_mode: null + key: bash-login + mount_path: /workspace/.bash_login + name: bash-login + persist: false + readonly: true + default_url: null + definition: + default: false + description: Understand the role of STAC in input/output data manifests in EO + data processing workflows + display_name: Understanding STAC for input/output data modelling + kubespawner_override: + cpu_guarantee: 1 + cpu_limit: 2 + extra_resource_guarantees: {} + extra_resource_limits: {} + image: docker.io/eoepca/pde-code-server@sha256:f57a3d5eabcae667e0db6e84a57b0c07c692c88f0fb5c8f6900ab8d5e38fcd40 + mem_guarantee: 4G + mem_limit: 6G + slug: eoepca_coder_slug_stac + groups: + - group-a + - group-b + id: profile_studio_coder_stac + image_pull_secrets: + - data: null + name: cr-config + persist: false + init_containers: + - command: + - sh + - -c + - sh /opt/init/.init.sh + image: eoepca/pde-code-server:develop + name: init-file-on-volume + volume_mounts: + - mount_path: /workspace + name: workspace-volume + - mount_path: /opt/init/.init.sh + name: init + sub_path: init + manifests: + - content: + - apiVersion: v1 + kind: ServiceAccount + metadata: + name: localstack + - apiVersion: rbac.authorization.k8s.io/v1 + kind: Role + metadata: + name: localstack + rules: + - apiGroups: + - '' + resources: + - pods + verbs: + - '*' + - apiGroups: + - '' + resources: + - pods/log + verbs: + - get + - apiGroups: + - '' + resources: + - pods/exec + verbs: + - get + - create + - apiGroups: + - '' + resources: + - services + verbs: + - get + - list + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + name: localstack + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: localstack + subjects: + - kind: ServiceAccount + name: localstack + - apiVersion: v1 + kind: Service + metadata: + name: localstack + spec: + ports: + - name: edge + port: 4566 + targetPort: 4566 + - name: external-service-port-4510 + port: 4510 + targetPort: ext-svc-4510 + - name: external-service-port-4511 + port: 4511 + targetPort: ext-svc-4511 + - name: external-service-port-4512 + port: 4512 + targetPort: ext-svc-4512 + - name: external-service-port-4513 + port: 4513 + targetPort: ext-svc-4513 + - name: external-service-port-4514 + port: 4514 + targetPort: ext-svc-4514 + - name: external-service-port-4515 + port: 4515 + targetPort: ext-svc-4515 + - name: external-service-port-4516 + port: 4516 + targetPort: ext-svc-4516 + - name: external-service-port-4517 + port: 4517 + targetPort: ext-svc-4517 + - name: external-service-port-4518 + port: 4518 + targetPort: ext-svc-4518 + - name: external-service-port-4519 + port: 4519 + targetPort: ext-svc-4519 + - name: external-service-port-4520 + port: 4520 + targetPort: ext-svc-4520 + - name: external-service-port-4521 + port: 4521 + targetPort: ext-svc-4521 + - name: external-service-port-4522 + port: 4522 + targetPort: ext-svc-4522 + - name: external-service-port-4523 + port: 4523 + targetPort: ext-svc-4523 + - name: external-service-port-4524 + port: 4524 + targetPort: ext-svc-4524 + - name: external-service-port-4525 + port: 4525 + targetPort: ext-svc-4525 + - name: external-service-port-4526 + port: 4526 + targetPort: ext-svc-4526 + - name: external-service-port-4527 + port: 4527 + targetPort: ext-svc-4527 + - name: external-service-port-4528 + port: 4528 + targetPort: ext-svc-4528 + - name: external-service-port-4529 + port: 4529 + targetPort: ext-svc-4529 + - name: external-service-port-4530 + port: 4530 + targetPort: ext-svc-4530 + - name: external-service-port-4531 + port: 4531 + targetPort: ext-svc-4531 + - name: external-service-port-4532 + port: 4532 + targetPort: ext-svc-4532 + - name: external-service-port-4533 + port: 4533 + targetPort: ext-svc-4533 + - name: external-service-port-4534 + port: 4534 + targetPort: ext-svc-4534 + - name: external-service-port-4535 + port: 4535 + targetPort: ext-svc-4535 + - name: external-service-port-4536 + port: 4536 + targetPort: ext-svc-4536 + - name: external-service-port-4537 + port: 4537 + targetPort: ext-svc-4537 + - name: external-service-port-4538 + port: 4538 + targetPort: ext-svc-4538 + - name: external-service-port-4539 + port: 4539 + targetPort: ext-svc-4539 + - name: external-service-port-4540 + port: 4540 + targetPort: ext-svc-4540 + - name: external-service-port-4541 + port: 4541 + targetPort: ext-svc-4541 + - name: external-service-port-4542 + port: 4542 + targetPort: ext-svc-4542 + - name: external-service-port-4543 + port: 4543 + targetPort: ext-svc-4543 + - name: external-service-port-4544 + port: 4544 + targetPort: ext-svc-4544 + - name: external-service-port-4545 + port: 4545 + targetPort: ext-svc-4545 + - name: external-service-port-4546 + port: 4546 + targetPort: ext-svc-4546 + - name: external-service-port-4547 + port: 4547 + targetPort: ext-svc-4547 + - name: external-service-port-4548 + port: 4548 + targetPort: ext-svc-4548 + - name: external-service-port-4549 + port: 4549 + targetPort: ext-svc-4549 + - name: external-service-port-4550 + port: 4550 + targetPort: ext-svc-4550 + - name: external-service-port-4551 + port: 4551 + targetPort: ext-svc-4551 + - name: external-service-port-4552 + port: 4552 + targetPort: ext-svc-4552 + - name: external-service-port-4553 + port: 4553 + targetPort: ext-svc-4553 + - name: external-service-port-4554 + port: 4554 + targetPort: ext-svc-4554 + - name: external-service-port-4555 + port: 4555 + targetPort: ext-svc-4555 + - name: external-service-port-4556 + port: 4556 + targetPort: ext-svc-4556 + - name: external-service-port-4557 + port: 4557 + targetPort: ext-svc-4557 + - name: external-service-port-4558 + port: 4558 + targetPort: ext-svc-4558 + - name: external-service-port-4559 + port: 4559 + targetPort: ext-svc-4559 + selector: + app.kubernetes.io/instance: localstack + app.kubernetes.io/name: localstack + type: ClusterIP + - apiVersion: apps/v1 + kind: Deployment + metadata: + name: localstack + spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/instance: localstack + app.kubernetes.io/name: localstack + strategy: + type: RollingUpdate + template: + metadata: + labels: + app.kubernetes.io/instance: localstack + app.kubernetes.io/name: localstack + spec: + containers: + - env: + - name: DEBUG + value: '0' + - name: EXTERNAL_SERVICE_PORTS_START + value: '4510' + - name: EXTERNAL_SERVICE_PORTS_END + value: '4560' + - name: LOCALSTACK_K8S_SERVICE_NAME + value: localstack + - name: LOCALSTACK_K8S_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: LAMBDA_RUNTIME_EXECUTOR + value: docker + - name: LAMBDA_K8S_IMAGE_PREFIX + value: localstack/lambda- + - name: LAMBDA_RUNTIME_ENVIRONMENT_TIMEOUT + value: '60' + - name: OVERRIDE_IN_DOCKER + value: '1' + image: localstack/localstack:latest + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 3 + httpGet: + path: /_localstack/health + port: edge + initialDelaySeconds: 0 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + name: localstack + ports: + - containerPort: 4566 + name: edge + protocol: TCP + - containerPort: 4510 + name: ext-svc-4510 + protocol: TCP + - containerPort: 4511 + name: ext-svc-4511 + protocol: TCP + - containerPort: 4512 + name: ext-svc-4512 + protocol: TCP + - containerPort: 4513 + name: ext-svc-4513 + protocol: TCP + - containerPort: 4514 + name: ext-svc-4514 + protocol: TCP + - containerPort: 4515 + name: ext-svc-4515 + protocol: TCP + - containerPort: 4516 + name: ext-svc-4516 + protocol: TCP + - containerPort: 4517 + name: ext-svc-4517 + protocol: TCP + - containerPort: 4518 + name: ext-svc-4518 + protocol: TCP + - containerPort: 4519 + name: ext-svc-4519 + protocol: TCP + - containerPort: 4520 + name: ext-svc-4520 + protocol: TCP + - containerPort: 4521 + name: ext-svc-4521 + protocol: TCP + - containerPort: 4522 + name: ext-svc-4522 + protocol: TCP + - containerPort: 4523 + name: ext-svc-4523 + protocol: TCP + - containerPort: 4524 + name: ext-svc-4524 + protocol: TCP + - containerPort: 4525 + name: ext-svc-4525 + protocol: TCP + - containerPort: 4526 + name: ext-svc-4526 + protocol: TCP + - containerPort: 4527 + name: ext-svc-4527 + protocol: TCP + - containerPort: 4528 + name: ext-svc-4528 + protocol: TCP + - containerPort: 4529 + name: ext-svc-4529 + protocol: TCP + - containerPort: 4530 + name: ext-svc-4530 + protocol: TCP + - containerPort: 4531 + name: ext-svc-4531 + protocol: TCP + - containerPort: 4532 + name: ext-svc-4532 + protocol: TCP + - containerPort: 4533 + name: ext-svc-4533 + protocol: TCP + - containerPort: 4534 + name: ext-svc-4534 + protocol: TCP + - containerPort: 4535 + name: ext-svc-4535 + protocol: TCP + - containerPort: 4536 + name: ext-svc-4536 + protocol: TCP + - containerPort: 4537 + name: ext-svc-4537 + protocol: TCP + - containerPort: 4538 + name: ext-svc-4538 + protocol: TCP + - containerPort: 4539 + name: ext-svc-4539 + protocol: TCP + - containerPort: 4540 + name: ext-svc-4540 + protocol: TCP + - containerPort: 4541 + name: ext-svc-4541 + protocol: TCP + - containerPort: 4542 + name: ext-svc-4542 + protocol: TCP + - containerPort: 4543 + name: ext-svc-4543 + protocol: TCP + - containerPort: 4544 + name: ext-svc-4544 + protocol: TCP + - containerPort: 4545 + name: ext-svc-4545 + protocol: TCP + - containerPort: 4546 + name: ext-svc-4546 + protocol: TCP + - containerPort: 4547 + name: ext-svc-4547 + protocol: TCP + - containerPort: 4548 + name: ext-svc-4548 + protocol: TCP + - containerPort: 4549 + name: ext-svc-4549 + protocol: TCP + - containerPort: 4550 + name: ext-svc-4550 + protocol: TCP + - containerPort: 4551 + name: ext-svc-4551 + protocol: TCP + - containerPort: 4552 + name: ext-svc-4552 + protocol: TCP + - containerPort: 4553 + name: ext-svc-4553 + protocol: TCP + - containerPort: 4554 + name: ext-svc-4554 + protocol: TCP + - containerPort: 4555 + name: ext-svc-4555 + protocol: TCP + - containerPort: 4556 + name: ext-svc-4556 + protocol: TCP + - containerPort: 4557 + name: ext-svc-4557 + protocol: TCP + - containerPort: 4558 + name: ext-svc-4558 + protocol: TCP + - containerPort: 4559 + name: ext-svc-4559 + protocol: TCP + readinessProbe: + failureThreshold: 3 + httpGet: + path: /_localstack/health + port: edge + initialDelaySeconds: 0 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + resources: {} + securityContext: {} + securityContext: {} + serviceAccountName: localstack + volumes: [] + key: manifests + name: manifests + persist: false + node_selector: {} + pod_env_vars: + AWS_ACCESS_KEY_ID: test + AWS_DEFAULT_REGION: us-east-1 + AWS_SECRET_ACCESS_KEY: test + CODE_SERVER_WS: /workspace/stac-eoap + CONDARC: /workspace/.condarc + CONDA_ENVS_PATH: /workspace/.envs + CWLTOOL_OPTIONS: --podman + HOME: /workspace + XDG_CONFIG_HOME: /workspace/.local + XDG_DATA_HOME: /workspace/.local/share/ + XDG_RUNTIME_DIR: /workspace/.local + role_bindings: [] + volumes: + - access_modes: + - ReadWriteOnce + claim_name: workspace-claim + name: workspace-volume + persist: false + size: 50Gi + storage_class: managed-nfs-storage + volume_mount: + mount_path: /workspace + name: workspace-volume +- config_maps: + - content: "mkdir -p /workspace/.config/autostart\n\n\ncat < /workspace/.config/autostart/qgis.desktop\ + \ \n[Desktop Entry]\nEncoding=UTF-8\nVersion=0.9.4\nType=Application\nName=qgis\n\ + Comment=qgis\nExec=qgis\nOnlyShowIn=XFCE;\nRunHook=0\nStartupNotify=false\n\ + Terminal=false\nHidden=false\nEOF" + default_mode: null + key: init + mount_path: /opt/init/.init.sh + name: init + persist: false + readonly: true + - content: "alias ll=\"ls -l\"\nalias calrissian=\"/opt/conda/bin/calrissian --pod-nodeselectors\ + \ /etc/calrissian/pod-node-selector.yml --stdout /calrissian/results.json --max-ram\ + \ 16G --max-cores \"8\" --tmp-outdir-prefix /calrissian/tmp/ --outdir /calrissian/\"\ + \nalias cwltool=\"/opt/conda/bin/cwltool --podman\"\n. /home/jovyan/.bashrc\n\ + \nalias aws=\"aws --endpoint-url=http://localstack:4566\"\n\n# >>> conda initialize\ + \ >>>\n# !! Contents within this block are managed by 'conda init' !!\n__conda_setup=\"\ + $('/opt/conda/bin/conda' 'shell.bash' 'hook' 2> /dev/null)\"\nif [ $? -eq 0\ + \ ]; then\n eval \"$__conda_setup\"\nelse\n if [ -f \"/opt/conda/etc/profile.d/conda.sh\"\ + \ ]; then\n . \"/opt/conda/etc/profile.d/conda.sh\"\n else\n \ + \ export PATH=\"/srv/conda/bin:$PATH\"\n fi\nfi\nunset __conda_setup\n\n\ + if [ -f \"/opt/conda/etc/profile.d/mamba.sh\" ]; then\n . \"/opt/conda/etc/profile.d/mamba.sh\"\ + \nfi\n# <<< conda initialize <<<\n" + default_mode: null + key: bash-rc + mount_path: /workspace/.bashrc + name: bash-rc + persist: false + readonly: true + - content: 'source /workspace/.bashrc + + ' + default_mode: null + key: bash-login + mount_path: /workspace/.bash_login + name: bash-login + persist: false + readonly: true + default_url: desktop + definition: + default: false + description: Spatial visualization and decision-making tools for everyone + display_name: QGIS on a Remote Desktop + kubespawner_override: + cpu_guarantee: null + cpu_limit: 2 + extra_resource_guarantees: {} + extra_resource_limits: {} + image: eoepca/iga-remote-desktop-qgis:1.1.2 + mem_guarantee: null + mem_limit: 2G + slug: eoepca_desktop_qgis + groups: + - group-a + - group-b + id: profile_studio_desktop_qgis + image_pull_secrets: [] + init_containers: + - command: + - sh + - -c + - sh /opt/init/.init.sh + image: eoepca/pde-code-server:develop + name: init-file-on-volume + volume_mounts: + - mount_path: /workspace + name: workspace-volume + - mount_path: /opt/init/.init.sh + name: init + sub_path: init + manifests: null + node_selector: {} + pod_env_vars: + AWS_ACCESS_KEY_ID: SCWMRGD5GA0Y68XRKFW8 + AWS_REGION: fr-par + AWS_S3_ENDPOINT: s3.fr-par.scw.cloud + AWS_SECRET_ACCESS_KEY: a9a7dc0c-8b0a-47b1-a311-b134b258dbb3 + AWS_VIRTUAL_HOSTING: 'true' + HOME: /workspace + role_bindings: null + volumes: + - access_modes: + - ReadWriteOnce + claim_name: workspace-claim + name: workspace-volume + persist: false + size: 50Gi + storage_class: managed-nfs-storage + volume_mount: + mount_path: /workspace + name: workspace-volume diff --git a/files/hub/config.yml b/files/hub/config.yml new file mode 100644 index 0000000..9d2021b --- /dev/null +++ b/files/hub/config.yml @@ -0,0 +1,787 @@ +profiles: +- config_maps: + - content: 'set -x + + + cd /workspace + + + git clone ''https://github.com/eoap/mastering-app-package.git'' + + + code-server --install-extension ms-python.python + + code-server --install-extension redhat.vscode-yaml + + code-server --install-extension sbg-rabix.benten-cwl + + code-server --install-extension ms-toolsai.jupyter + + + ln -s /workspace/.local/share/code-server/extensions /workspace/extensions + + + exit 0 + + ' + default_mode: '0660' + key: init + mount_path: /opt/init/.init.sh + name: init + persist: false + readonly: true + - content: null + default_mode: null + key: gateway + mount_path: /etc/dask/gateway.yaml + name: dask-gateway-config + persist: true + readonly: true + - content: null + default_mode: null + key: gateway + mount_path: /workspace/.config/dask/gateway.yaml + name: dask-gateway-config-ws + persist: true + readonly: true + - content: 'source /workspace/.bashrc + + ' + default_mode: null + key: bash-login + mount_path: /etc/profile.d/bash-login.sh + name: bash-login + persist: true + readonly: true + - content: "alias ll=\"ls -l\"\nalias calrissian=\"/opt/conda/bin/calrissian --pod-nodeselectors /etc/calrissian/pod-node-selector.yml --stdout /calrissian/results.json --max-ram 16G --max-cores \"8\"\ + \ --tmp-outdir-prefix /calrissian/tmp/ --outdir /calrissian/\"\nalias cwltool=\"/opt/conda/bin/cwltool --podman\"\n. /home/jovyan/.bashrc\n\n#alias aws=\"aws --endpoint-url=http://localstack:4566\"\ + \n\n# >>> conda initialize >>>\n# !! Contents within this block are managed by 'conda init' !!\n__conda_setup=\"$('/opt/conda/bin/conda' 'shell.bash' 'hook' 2> /dev/null)\"\nif [ $? -eq 0 ]; then\n\ + \ eval \"$__conda_setup\"\nelse\n if [ -f \"/opt/conda/etc/profile.d/conda.sh\" ]; then\n . \"/opt/conda/etc/profile.d/conda.sh\"\n else\n export PATH=\"/srv/conda/bin:$PATH\"\ + \n fi\nfi\nunset __conda_setup\n\nif [ -f \"/opt/conda/etc/profile.d/mamba.sh\" ]; then\n . \"/opt/conda/etc/profile.d/mamba.sh\"\nfi\n# <<< conda initialize <<<\n\na={{spawner.user.name}}\n\ + \nalias aws=\"aws --endpoint-url=http://localstack-jupyter-{{spawner.user.name}}:4566\"" + default_mode: null + key: bash-rc + mount_path: /workspace/.bashrc + name: bash-rc + persist: true + readonly: true + default_url: null + definition: + default: false + description: This profile is used to demonstrate the use of an init script + display_name: Coder demo init script + kubespawner_override: + cpu_guarantee: 1 + cpu_limit: 2 + extra_resource_guarantees: {} + extra_resource_limits: {} + image: eoepca/pde-code-server:develop + mem_guarantee: 4G + mem_limit: 6G + slug: profile_1 + env_from_config_maps: + - my-config + env_from_secrets: + - my-secret + - data-by-name + groups: + - group-a + - group-b + id: profile_1 + image_pull_secrets: + - data: null + name: eoepca-plus-secret-ro + persist: true + init_containers: + - command: + - sh + - -c + - sh /opt/init/.init.sh + image: eoepca/pde-code-server:develop + name: init-file-on-volume + volume_mounts: + - mount_path: /calrissian + name: workspace-volume + - mount_path: /opt/init/.init.sh + name: init + sub_path: init + manifests: + - content: + - apiVersion: v1 + kind: ServiceAccount + metadata: + name: localstack + - apiVersion: rbac.authorization.k8s.io/v1 + kind: Role + metadata: + name: localstack + rules: + - apiGroups: + - '' + resources: + - pods + verbs: + - '*' + - apiGroups: + - '' + resources: + - pods/log + verbs: + - get + - apiGroups: + - '' + resources: + - pods/exec + verbs: + - get + - create + - apiGroups: + - '' + resources: + - services + verbs: + - get + - list + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + name: localstack + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: localstack + subjects: + - kind: ServiceAccount + name: localstack + - apiVersion: v1 + kind: Service + metadata: + name: localstack + spec: + ports: + - name: edge + port: 4566 + targetPort: 4566 + - name: external-service-port-4510 + port: 4510 + targetPort: ext-svc-4510 + - name: external-service-port-4511 + port: 4511 + targetPort: ext-svc-4511 + - name: external-service-port-4512 + port: 4512 + targetPort: ext-svc-4512 + - name: external-service-port-4513 + port: 4513 + targetPort: ext-svc-4513 + - name: external-service-port-4514 + port: 4514 + targetPort: ext-svc-4514 + - name: external-service-port-4515 + port: 4515 + targetPort: ext-svc-4515 + - name: external-service-port-4516 + port: 4516 + targetPort: ext-svc-4516 + - name: external-service-port-4517 + port: 4517 + targetPort: ext-svc-4517 + - name: external-service-port-4518 + port: 4518 + targetPort: ext-svc-4518 + - name: external-service-port-4519 + port: 4519 + targetPort: ext-svc-4519 + - name: external-service-port-4520 + port: 4520 + targetPort: ext-svc-4520 + - name: external-service-port-4521 + port: 4521 + targetPort: ext-svc-4521 + - name: external-service-port-4522 + port: 4522 + targetPort: ext-svc-4522 + - name: external-service-port-4523 + port: 4523 + targetPort: ext-svc-4523 + - name: external-service-port-4524 + port: 4524 + targetPort: ext-svc-4524 + - name: external-service-port-4525 + port: 4525 + targetPort: ext-svc-4525 + - name: external-service-port-4526 + port: 4526 + targetPort: ext-svc-4526 + - name: external-service-port-4527 + port: 4527 + targetPort: ext-svc-4527 + - name: external-service-port-4528 + port: 4528 + targetPort: ext-svc-4528 + - name: external-service-port-4529 + port: 4529 + targetPort: ext-svc-4529 + - name: external-service-port-4530 + port: 4530 + targetPort: ext-svc-4530 + - name: external-service-port-4531 + port: 4531 + targetPort: ext-svc-4531 + - name: external-service-port-4532 + port: 4532 + targetPort: ext-svc-4532 + - name: external-service-port-4533 + port: 4533 + targetPort: ext-svc-4533 + - name: external-service-port-4534 + port: 4534 + targetPort: ext-svc-4534 + - name: external-service-port-4535 + port: 4535 + targetPort: ext-svc-4535 + - name: external-service-port-4536 + port: 4536 + targetPort: ext-svc-4536 + - name: external-service-port-4537 + port: 4537 + targetPort: ext-svc-4537 + - name: external-service-port-4538 + port: 4538 + targetPort: ext-svc-4538 + - name: external-service-port-4539 + port: 4539 + targetPort: ext-svc-4539 + - name: external-service-port-4540 + port: 4540 + targetPort: ext-svc-4540 + - name: external-service-port-4541 + port: 4541 + targetPort: ext-svc-4541 + - name: external-service-port-4542 + port: 4542 + targetPort: ext-svc-4542 + - name: external-service-port-4543 + port: 4543 + targetPort: ext-svc-4543 + - name: external-service-port-4544 + port: 4544 + targetPort: ext-svc-4544 + - name: external-service-port-4545 + port: 4545 + targetPort: ext-svc-4545 + - name: external-service-port-4546 + port: 4546 + targetPort: ext-svc-4546 + - name: external-service-port-4547 + port: 4547 + targetPort: ext-svc-4547 + - name: external-service-port-4548 + port: 4548 + targetPort: ext-svc-4548 + - name: external-service-port-4549 + port: 4549 + targetPort: ext-svc-4549 + - name: external-service-port-4550 + port: 4550 + targetPort: ext-svc-4550 + - name: external-service-port-4551 + port: 4551 + targetPort: ext-svc-4551 + - name: external-service-port-4552 + port: 4552 + targetPort: ext-svc-4552 + - name: external-service-port-4553 + port: 4553 + targetPort: ext-svc-4553 + - name: external-service-port-4554 + port: 4554 + targetPort: ext-svc-4554 + - name: external-service-port-4555 + port: 4555 + targetPort: ext-svc-4555 + - name: external-service-port-4556 + port: 4556 + targetPort: ext-svc-4556 + - name: external-service-port-4557 + port: 4557 + targetPort: ext-svc-4557 + - name: external-service-port-4558 + port: 4558 + targetPort: ext-svc-4558 + - name: external-service-port-4559 + port: 4559 + targetPort: ext-svc-4559 + selector: + app.kubernetes.io/instance: localstack + app.kubernetes.io/name: localstack + type: ClusterIP + - apiVersion: apps/v1 + kind: Deployment + metadata: + name: localstack + spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/instance: localstack + app.kubernetes.io/name: localstack + strategy: + type: RollingUpdate + template: + metadata: + labels: + app: localstack-{{ spawner.user.name }} + app.kubernetes.io/instance: localstack + app.kubernetes.io/name: localstack + spec: + containers: + - env: + - name: DEBUG + value: '0' + - name: EXTERNAL_SERVICE_PORTS_START + value: '4510' + - name: EXTERNAL_SERVICE_PORTS_END + value: '4560' + - name: LOCALSTACK_K8S_SERVICE_NAME + value: localstack + - name: LOCALSTACK_K8S_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: LAMBDA_RUNTIME_EXECUTOR + value: docker + - name: LAMBDA_K8S_IMAGE_PREFIX + value: localstack/lambda- + - name: LAMBDA_RUNTIME_ENVIRONMENT_TIMEOUT + value: '60' + - name: OVERRIDE_IN_DOCKER + value: '1' + image: localstack/localstack:latest + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 3 + httpGet: + path: /_localstack/health + port: edge + initialDelaySeconds: 0 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + name: localstack + ports: + - containerPort: 4566 + name: edge + protocol: TCP + - containerPort: 4510 + name: ext-svc-4510 + protocol: TCP + - containerPort: 4511 + name: ext-svc-4511 + protocol: TCP + - containerPort: 4512 + name: ext-svc-4512 + protocol: TCP + - containerPort: 4513 + name: ext-svc-4513 + protocol: TCP + - containerPort: 4514 + name: ext-svc-4514 + protocol: TCP + - containerPort: 4515 + name: ext-svc-4515 + protocol: TCP + - containerPort: 4516 + name: ext-svc-4516 + protocol: TCP + - containerPort: 4517 + name: ext-svc-4517 + protocol: TCP + - containerPort: 4518 + name: ext-svc-4518 + protocol: TCP + - containerPort: 4519 + name: ext-svc-4519 + protocol: TCP + - containerPort: 4520 + name: ext-svc-4520 + protocol: TCP + - containerPort: 4521 + name: ext-svc-4521 + protocol: TCP + - containerPort: 4522 + name: ext-svc-4522 + protocol: TCP + - containerPort: 4523 + name: ext-svc-4523 + protocol: TCP + - containerPort: 4524 + name: ext-svc-4524 + protocol: TCP + - containerPort: 4525 + name: ext-svc-4525 + protocol: TCP + - containerPort: 4526 + name: ext-svc-4526 + protocol: TCP + - containerPort: 4527 + name: ext-svc-4527 + protocol: TCP + - containerPort: 4528 + name: ext-svc-4528 + protocol: TCP + - containerPort: 4529 + name: ext-svc-4529 + protocol: TCP + - containerPort: 4530 + name: ext-svc-4530 + protocol: TCP + - containerPort: 4531 + name: ext-svc-4531 + protocol: TCP + - containerPort: 4532 + name: ext-svc-4532 + protocol: TCP + - containerPort: 4533 + name: ext-svc-4533 + protocol: TCP + - containerPort: 4534 + name: ext-svc-4534 + protocol: TCP + - containerPort: 4535 + name: ext-svc-4535 + protocol: TCP + - containerPort: 4536 + name: ext-svc-4536 + protocol: TCP + - containerPort: 4537 + name: ext-svc-4537 + protocol: TCP + - containerPort: 4538 + name: ext-svc-4538 + protocol: TCP + - containerPort: 4539 + name: ext-svc-4539 + protocol: TCP + - containerPort: 4540 + name: ext-svc-4540 + protocol: TCP + - containerPort: 4541 + name: ext-svc-4541 + protocol: TCP + - containerPort: 4542 + name: ext-svc-4542 + protocol: TCP + - containerPort: 4543 + name: ext-svc-4543 + protocol: TCP + - containerPort: 4544 + name: ext-svc-4544 + protocol: TCP + - containerPort: 4545 + name: ext-svc-4545 + protocol: TCP + - containerPort: 4546 + name: ext-svc-4546 + protocol: TCP + - containerPort: 4547 + name: ext-svc-4547 + protocol: TCP + - containerPort: 4548 + name: ext-svc-4548 + protocol: TCP + - containerPort: 4549 + name: ext-svc-4549 + protocol: TCP + - containerPort: 4550 + name: ext-svc-4550 + protocol: TCP + - containerPort: 4551 + name: ext-svc-4551 + protocol: TCP + - containerPort: 4552 + name: ext-svc-4552 + protocol: TCP + - containerPort: 4553 + name: ext-svc-4553 + protocol: TCP + - containerPort: 4554 + name: ext-svc-4554 + protocol: TCP + - containerPort: 4555 + name: ext-svc-4555 + protocol: TCP + - containerPort: 4556 + name: ext-svc-4556 + protocol: TCP + - containerPort: 4557 + name: ext-svc-4557 + protocol: TCP + - containerPort: 4558 + name: ext-svc-4558 + protocol: TCP + - containerPort: 4559 + name: ext-svc-4559 + protocol: TCP + readinessProbe: + failureThreshold: 3 + httpGet: + path: /_localstack/health + port: edge + initialDelaySeconds: 0 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + resources: {} + securityContext: {} + securityContext: {} + serviceAccountName: localstack + volumes: [] + - apiVersion: v1 + data: + ENV_VAR1: value1 + ENV_VAR2: value2 + kind: ConfigMap + metadata: + name: my-config + - apiVersion: v1 + data: + SECRET_KEY1: dmFsdWUx + SECRET_KEY2: dmFsdWUy + kind: Secret + metadata: + name: my-secret + type: Opaque + - apiVersion: v1 + data: + credentials: W2RlZmF1bHRdCmF3c19hY2Nlc3Nfa2V5X2lkPUFTSUFJT1NGT0ROTjdFWEFNUExFCmF3c19zZWNyZXRfYWNjZXNzX2tleT13SmFsclhVdG5GRU1JL0s3TURFTkcvYlB4UmZpQ1lFWEFNUExFS0VZCmF3c19zZXNzaW9uX3Rva2VuPUlRb0piM2pySmdCV05FTE5Hb2xHSkxFT3RTVEFOR1k0TFlPNUk0SzVOUlZFS1pPTkNTTk1HRlNUS1FNSUxXUjJPUzAwRklDRTExSlg= + kind: Secret + metadata: + name: aws-credentials-{{ spawner.user.name }} + type: Opaque + - apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret + metadata: + name: data-by-name + namespace: jupyter-{{ spawner.user.name }} + spec: + data: + - remoteRef: + key: secret-one + property: the-key + secretKey: secret-value + refreshInterval: 15s + secretStoreRef: + kind: ClusterSecretStore + name: k8s-secret-store + target: + creationPolicy: Owner + name: data-by-name + - apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret + metadata: + name: eoepca-plus-secret-ro + namespace: jupyter-{{ spawner.user.name }} + spec: + data: + - remoteRef: + key: eoepca-plus-secret-ro + property: .dockerconfigjson + secretKey: .dockerconfigjson + refreshInterval: 15s + secretStoreRef: + kind: ClusterSecretStore + name: k8s-secret-store + target: + creationPolicy: Owner + name: eoepca-plus-secret-ro + key: localstack + name: localstack + persist: false + - content: + - apiVersion: helm.crossplane.io/v1beta1 + kind: Release + metadata: + name: dask-gw-jupyter-{{ spawner.user.name }} + namespace: jupyter-{{ spawner.user.name }} + spec: + forProvider: + chart: + name: dask-gateway + repository: https://helm.dask.org + version: 2024.1.0 + namespace: jupyter-{{ spawner.user.name }} + values: + gateway: + backend: + image: + name: ghcr.io/fabricebrito/dev-platform-dask-gateway/worker + tag: 1.0.0 + extraConfig: + dask_gateway_config.py: "c = get_config()\nfrom dask_gateway_server.options import Options, String, Integer, Float\nc.Backend.cluster_options = Options(\n Float(\"worker_cores_limit\",\ + \ default=1, label=\"Worker Cores Limit\"),\n Float(\"worker_cores\", default=1, label=\"Worker Cores\"),\n String(\"worker_memory\", default=\"1 G\", label=\"Worker Memory\"),\n String(\"\ + image\", default=\"daskgateway/dask-worker:latest\", label=\"Worker Image\")\n)\n" + traefik: + service: + type: ClusterIP + providerConfigRef: + name: helm-provider + - apiVersion: v1 + data: + gateway: "gateway:\n address: http://traefik-dask-gw-jupyter-{{ spawner.user.name }}-dask-gateway.jupyter-{{ spawner.user.name }}.svc.cluster.local:80\n\n cluster:\n options: \n image:\ + \ \"ghcr.io/fabricebrito/dev-platform-dask-gateway/worker:1.0.0\"\n worker_cores: 0.5\n worker_cores_limit: 1\n worker_memory: \"4 G\"\n" + kind: ConfigMap + metadata: + name: dask-gateway-config + - apiVersion: v1 + data: + gateway: "gateway:\n address: http://traefik-dask-gw-jupyter-{{ spawner.user.name }}-dask-gateway.jupyter-{{ spawner.user.name }}.svc.cluster.local:80\n\n cluster:\n options: \n image:\ + \ \"ghcr.io/fabricebrito/dev-platform-dask-gateway/worker:1.0.0\"\n worker_cores: 0.5\n worker_cores_limit: 1\n worker_memory: \"4 G\"\n" + kind: ConfigMap + metadata: + name: dask-gateway-config-ws + - apiVersion: rbac.authorization.k8s.io/v1 + kind: Role + metadata: + name: access-services + namespace: jupyter-{{ spawner.user.name }} + rules: + - apiGroups: + - '' + resources: + - services + - pods/exec + verbs: + - get + - list + - watch + - apiGroups: + - '' + resources: + - pods + verbs: + - get + - list + - watch + - apiGroups: + - '' + resources: + - pods/portforward + verbs: + - create + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + name: bind-default-to-services + namespace: jupyter-{{ spawner.user.name }} + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: access-services + subjects: + - kind: ServiceAccount + name: default + namespace: jupyter-{{ spawner.user.name }} + key: dask-gateway + name: dask-gateway + persist: false + - content: + - apiVersion: v1 + kind: Pod + metadata: + name: kaniko-build + namespace: jupyter-{{ spawner.user.name }} + spec: + containers: + - command: + - /bin/sh + - -c + - sleep infinity + image: gcr.io/kaniko-project/executor:debug + name: kaniko + volumeMounts: + - mountPath: /calrissian + name: build-context + - mountPath: /kaniko/.docker + name: kaniko-secret + restartPolicy: Never + volumes: + - name: build-context + persistentVolumeClaim: + claimName: calrissian-claim + - name: kaniko-secret + secret: + items: + - key: .dockerconfigjson + path: config.json + secretName: kaniko-secret + - apiVersion: rbac.authorization.k8s.io/v1 + kind: Role + metadata: + name: pod-exec + namespace: jupyter-{{ spawner.user.name }} + rules: + - apiGroups: + - '' + resources: + - pods/exec + verbs: + - get + - list + - watch + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + name: bind-default-to-opd-exec + namespace: jupyter-{{ spawner.user.name }} + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: pod-exec + subjects: + - kind: ServiceAccount + name: default + namespace: jupyter-{{ spawner.user.name }} + key: kaniko + name: kaniko + persist: false + node_selector: {} + pod_env_vars: + CODE_SERVER_WS: /workspace/mastering-app-package + CONDARC: /workspace/.condarc + CONDA_ENVS_PATH: /workspace/.envs + DASK_GATEWAY: http://traefik-dask-gw-jupyter-{{ spawner.user.name }}-dask-gateway.jupyter-{{ spawner.user.name }}.svc.cluster.local:80 + HOME: /workspace + XDG_RUNTIME_DIR: /workspace/.local + role_bindings: null + secret_mounts: + - mount_path: /workspace/.aws + name: aws-credentials-{{ spawner.user.name }} + sub_path: null + - mount_path: /workspace/.data-by-name + name: data-by-name + sub_path: null + - mount_path: /workspace/.docker/config.json + name: eoepca-plus-secret-ro + sub_path: .dockerconfigjson + volumes: + - access_modes: + - ReadWriteMany + claim_name: calrissian-claim + name: calrissian-volume + persist: false + size: 50Gi + storage_class: standard + volume_mount: + mount_path: /calrissian + name: calrissian-volume + - access_modes: + - ReadWriteOnce + claim_name: workspace-claim + name: workspace-volume + persist: true + size: 50Gi + storage_class: standard + volume_mount: + mount_path: /workspace + name: workspace-volume diff --git a/files/hub/jupyterhub_config.py b/files/hub/jupyterhub_config.py new file mode 100644 index 0000000..44fdd69 --- /dev/null +++ b/files/hub/jupyterhub_config.py @@ -0,0 +1,179 @@ +import os +import sys + +from tornado.httpclient import AsyncHTTPClient + + +from application_hub_context.app_hub_context import DefaultApplicationHubContext + +configuration_directory = os.path.dirname(os.path.realpath(__file__)) +sys.path.insert(0, configuration_directory) + +from z2jh import ( + get_config, + get_name, + get_name_env, +) + +config_path = "/usr/local/etc/applicationhub/config.yml" + +namespace_prefix = "jupyter" + + +def custom_options_form(spawner): + + spawner.log.info("Configure profile list") + + namespace = f"{namespace_prefix}-{spawner.user.name}" + + workspace = DefaultApplicationHubContext( + namespace=namespace, + spawner=spawner, + config_path=config_path, + ) + + spawner.profile_list = workspace.get_profile_list() + + return spawner._options_form_default() + + +def pre_spawn_hook(spawner): + + profile_slug = spawner.user_options.get("profile", None) + + env = os.environ["JUPYTERHUB_ENV"].lower() + + spawner.environment["CALRISSIAN_POD_NAME"] = f"jupyter-{spawner.user.name}-{env}" + + spawner.log.info(f"Using profile slug {profile_slug}") + + namespace = f"{namespace_prefix}-{spawner.user.name}" + + workspace = DefaultApplicationHubContext( + namespace=namespace, spawner=spawner, config_path=config_path, skip_namespace_check=False, + ) + + workspace.initialise() + + profile_id = workspace.config_parser.get_profile_by_slug(slug=profile_slug).id + + default_url = workspace.config_parser.get_profile_default_url(profile_id=profile_id) + + if default_url: + spawner.log.info(f"Setting default url to {default_url}") + spawner.default_url = default_url + + +def post_stop_hook(spawner): + + namespace = f"jupyter-{spawner.user.name}" + + workspace = DefaultApplicationHubContext( + namespace=namespace, spawner=spawner, config_path=config_path + ) + spawner.log.info("Dispose in post stop hook") + workspace.dispose() + + +c.JupyterHub.default_url = "spawn" + + +# Configure JupyterHub to use the curl backend for making HTTP requests, +# rather than the pure-python implementations. The default one starts +# being too slow to make a large number of requests to the proxy API +# at the rate required. +AsyncHTTPClient.configure("tornado.curl_httpclient.CurlAsyncHTTPClient") + +c.ConfigurableHTTPProxy.api_url = ( + f'http://{get_name("proxy-api")}:{get_name_env("proxy-api", "_SERVICE_PORT")}' +) +# c.ConfigurableHTTPProxy.should_start = False + +# Don't wait at all before redirecting a spawning user to the progress page +c.JupyterHub.tornado_settings = { + "slow_spawn_timeout": 0, +} + +jupyterhub_env = os.environ["JUPYTERHUB_ENV"].upper() +jupyterhub_hub_host = "application-hub-hub.jupyter" +jupyterhub_single_user_image = os.environ["JUPYTERHUB_SINGLE_USER_IMAGE_NOTEBOOKS"] + +# Authentication +c.LocalAuthenticator.create_system_users = True +c.Authenticator.admin_users = {"jovyan"} +# Deprecated +c.Authenticator.allowed_users = {"jovyan", "alice", "bob"} +c.JupyterHub.authenticator_class = "dummy" + +# HTTP Proxy auth token +c.ConfigurableHTTPProxy.auth_token = get_config("proxy.secretToken") +c.JupyterHub.cookie_secret_file = "/srv/jupyterhub/cookie_secret" +# Proxy config +c.JupyterHub.cleanup_servers = False +# Network +c.JupyterHub.allow_named_servers = True +c.JupyterHub.ip = "0.0.0.0" +c.JupyterHub.hub_ip = "0.0.0.0" +c.JupyterHub.hub_connect_ip = jupyterhub_hub_host +# Misc +c.JupyterHub.cleanup_servers = False + +# Culling +c.JupyterHub.services = [ + { + "name": "idle-culler", + "admin": True, + "command": [sys.executable, "-m", "jupyterhub_idle_culler", "--timeout=3600"], + } +] + +# Logs +c.JupyterHub.log_level = "DEBUG" + +# Spawner +c.JupyterHub.spawner_class = "kubespawner.KubeSpawner" +c.KubeSpawner.environment = { + "JUPYTER_ENABLE_LAB": "true", +} + +c.KubeSpawner.uid = 1001 +c.KubeSpawner.fs_gid = 100 +c.KubeSpawner.hub_connect_ip = jupyterhub_hub_host + +# SecurityContext +c.KubeSpawner.privileged = True +c.KubeSpawner.allow_privilege_escalation = True + +# ServiceAccount +c.KubeSpawner.service_account = "default" +c.KubeSpawner.start_timeout = 60 * 15 +c.KubeSpawner.image = jupyterhub_single_user_image +c.KubernetesSpawner.verify_ssl = True +c.KubeSpawner.pod_name_template = ( + "jupyter-{username}-{servername}-" + os.environ["JUPYTERHUB_ENV"].lower() +) + +# Namespace +c.KubeSpawner.namespace = "jupyter" + +# User namespace +c.KubeSpawner.enable_user_namespaces = True +c.KubeSpawner.user_namespace_labels = {"eso": "enabled"} + +# Volumes +# volumes are managed by the pre_spawn_hook/post_stop_hook + +# TODO - move this value to the values.yaml file +c.KubeSpawner.image_pull_secrets = ["cr-config"] + +# custom options form +c.KubeSpawner.options_form = custom_options_form + +# hooks +c.KubeSpawner.pre_spawn_hook = pre_spawn_hook +c.KubeSpawner.post_stop_hook = post_stop_hook + +c.JupyterHub.template_paths = [ + "/opt/jupyterhub/template", + "/usr/local/share/jupyterhub/templates", +] diff --git a/files/hub/z2jh.py b/files/hub/z2jh.py new file mode 100644 index 0000000..57e463f --- /dev/null +++ b/files/hub/z2jh.py @@ -0,0 +1,121 @@ +""" +Utility methods for use in jupyterhub_config.py and dynamic subconfigs. + +Methods here can be imported by extraConfig in values.yaml +""" +# from collections import Mapping +from collections.abc import Mapping +from functools import lru_cache +import os + +import yaml + +# memoize so we only load config once +@lru_cache() +def _load_config(): + """Load the Helm chart configuration used to render the Helm templates of + the chart from a mounted k8s Secret, and merge in values from an optionally + mounted secret (hub.existingSecret).""" + + cfg = {} + for source in ("secret/values.yaml", "existing-secret/values.yaml"): + path = f"/usr/local/etc/jupyterhub/{source}" + if os.path.exists(path): + print(f"Loading {path}") + with open(path) as f: + values = yaml.safe_load(f) + cfg = _merge_dictionaries(cfg, values) + else: + print(f"No config at {path}") + return cfg + + +@lru_cache() +def _get_config_value(key): + """Load value from the k8s ConfigMap given a key.""" + + path = f"/usr/local/etc/jupyterhub/config/{key}" + if os.path.exists(path): + with open(path) as f: + return f.read() + else: + raise Exception(f"{path} not found!") + + +@lru_cache() +def get_secret_value(key, default="never-explicitly-set"): + """Load value from the user managed k8s Secret or the default k8s Secret + given a key.""" + + for source in ("existing-secret", "secret"): + path = f"/usr/local/etc/jupyterhub/{source}/{key}" + if os.path.exists(path): + with open(path) as f: + return f.read() + if default != "never-explicitly-set": + return default + raise Exception(f"{key} not found in either k8s Secret!") + + +def get_name(name): + """Returns the fullname of a resource given its short name""" + return _get_config_value(name) + + +def get_name_env(name, suffix=""): + """Returns the fullname of a resource given its short name along with a + suffix, converted to uppercase with dashes replaced with underscores. This + is useful to reference named services associated environment variables, such + as PROXY_PUBLIC_SERVICE_PORT.""" + env_key = _get_config_value(name) + suffix + env_key = env_key.upper().replace("-", "_") + return os.environ[env_key] + + +def _merge_dictionaries(a, b): + """Merge two dictionaries recursively. + + Simplified From https://stackoverflow.com/a/7205107 + """ + merged = a.copy() + for key in b: + if key in a: + if isinstance(a[key], Mapping) and isinstance(b[key], Mapping): + merged[key] = _merge_dictionaries(a[key], b[key]) + else: + merged[key] = b[key] + else: + merged[key] = b[key] + return merged + + +def get_config(key, default=None): + """ + Find a config item of a given name & return it + + Parses everything as YAML, so lists and dicts are available too + + get_config("a.b.c") returns config['a']['b']['c'] + """ + value = _load_config() + # resolve path in yaml + for level in key.split("."): + if not isinstance(value, dict): + # a parent is a scalar or null, + # can't resolve full path + return default + if level not in value: + return default + else: + value = value[level] + return value + + +def set_config_if_not_none(cparent, name, key): + """ + Find a config item of a given name, set the corresponding Jupyter + configuration item if not None + """ + data = get_config(key) + if data is not None: + setattr(cparent, name, data) diff --git a/files/theme/page.html b/files/theme/page.html new file mode 100644 index 0000000..6ee147a --- /dev/null +++ b/files/theme/page.html @@ -0,0 +1,5 @@ +{% extends "templates/page.html" %} + +{% set announcement = 'EOEPCA+ ApplicationHub demonstration instance' %} + +{% block title %}ApplicationHub{% endblock %} diff --git a/files/theme/spawn.html b/files/theme/spawn.html new file mode 100644 index 0000000..8bb7896 --- /dev/null +++ b/files/theme/spawn.html @@ -0,0 +1,7 @@ +{% extends "templates/spawn.html" %} + +{% block heading %} +
+

Available EOEPCA+ demonstration applications

+
+{% endblock %} diff --git a/files/theme/spawn_pending.html b/files/theme/spawn_pending.html new file mode 100644 index 0000000..3cc1dfb --- /dev/null +++ b/files/theme/spawn_pending.html @@ -0,0 +1,94 @@ +{% extends "page.html" %} + +{% block main %} + +
+
+
+ {% block message %} +

Your EOEPCA application is starting up.

+

You will be redirected automatically when it's ready for you.

+ {% endblock %} +
+
+ 0% Complete +
+
+

+
+
+
+
+
+ Event log +
+
+
+
+
+ +{% endblock %} + +{% block script %} +{{ `{{ super() }} `}} + +{% endblock %} diff --git a/requirements.in b/requirements.in deleted file mode 100644 index 1e7680c..0000000 --- a/requirements.in +++ /dev/null @@ -1,24 +0,0 @@ -# The top-level dependencies are listed here. -# Using pip-tools you can automatically generate requirements.txt. -# This helps you segegate the top-level dependencies and makes the -# upgrade process easier. - -# Here, the dependencies aren't pinned to keep them updated. -# However, you should pin your top-level dependencies. - -# https://github.com/jazzband/pip-tools - -pyjwt -jupyterhub -pytest -pytest-asyncio -pytest-cov -requests-mock -jupyterhub-kubespawner -httplib2 -oauthenticator -jupyterhub-idle-culler -kubernetes -loguru -addict -pydantic>=2 diff --git a/requirements.txt b/requirements.txt index 8c2f081..e35fc56 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,238 +1,2 @@ -# -# This file is autogenerated by pip-compile with python 3.8 -# To update, run: -# -# pip-compile requirements.in -# -addict==2.4.0 - # via -r requirements.in -aiohttp==3.8.4 - # via kubernetes-asyncio -aiosignal==1.3.1 - # via aiohttp -alembic==1.11.1 - # via jupyterhub -annotated-types==0.6.0 - # via pydantic -async-generator==1.10 - # via jupyterhub -async-timeout==4.0.2 - # via aiohttp -attrs==23.1.0 - # via - # aiohttp - # jsonschema -cachetools==5.3.1 - # via google-auth -certifi==2023.5.7 - # via - # kubernetes - # kubernetes-asyncio - # requests -certipy==0.1.3 - # via jupyterhub -cffi==1.15.1 - # via cryptography -charset-normalizer==3.1.0 - # via - # aiohttp - # requests -coverage[toml]==7.2.7 - # via pytest-cov -cryptography==41.0.1 - # via pyopenssl -escapism==1.0.1 - # via jupyterhub-kubespawner -exceptiongroup==1.1.1 - # via pytest -frozenlist==1.3.3 - # via - # aiohttp - # aiosignal -google-auth==2.19.1 - # via kubernetes -greenlet==2.0.2 - # via sqlalchemy -httplib2==0.22.0 - # via -r requirements.in -idna==3.4 - # via - # requests - # yarl -importlib-metadata==6.8.0 - # via - # alembic - # jupyterhub -importlib-resources==6.1.1 - # via - # alembic - # jsonschema -iniconfig==2.0.0 - # via pytest -jinja2==3.1.2 - # via - # jupyterhub - # jupyterhub-kubespawner -jsonschema==4.17.3 - # via - # jupyter-telemetry - # oauthenticator -jupyter-telemetry==0.1.0 - # via jupyterhub -jupyterhub==4.0.0 - # via - # -r requirements.in - # jupyterhub-kubespawner - # oauthenticator -jupyterhub-idle-culler==1.2.1 - # via -r requirements.in -jupyterhub-kubespawner==6.0.0 - # via -r requirements.in -kubernetes==26.1.0 - # via -r requirements.in -kubernetes-asyncio==24.2.3 - # via jupyterhub-kubespawner -loguru==0.7.0 - # via -r requirements.in -mako==1.2.4 - # via alembic -markupsafe==2.1.3 - # via - # jinja2 - # mako -multidict==6.0.4 - # via - # aiohttp - # yarl -oauthenticator==15.1.0 - # via -r requirements.in -oauthlib==3.2.2 - # via - # jupyterhub - # requests-oauthlib -packaging==23.1 - # via - # jupyterhub - # pytest -pamela==1.1.0 - # via jupyterhub -pkgutil-resolve-name==1.3.10 - # via jsonschema -pluggy==1.0.0 - # via pytest -prometheus-client==0.17.0 - # via jupyterhub -pyasn1==0.5.0 - # via - # pyasn1-modules - # rsa -pyasn1-modules==0.3.0 - # via google-auth -pycparser==2.21 - # via cffi -pydantic==2.4.2 - # via -r requirements.in -pydantic-core==2.10.1 - # via pydantic -pyjwt==2.7.0 - # via -r requirements.in -pyopenssl==23.2.0 - # via certipy -pyparsing==3.0.9 - # via httplib2 -pyrsistent==0.19.3 - # via jsonschema -pytest==7.3.1 - # via - # -r requirements.in - # pytest-asyncio - # pytest-cov -pytest-asyncio==0.21.0 - # via -r requirements.in -pytest-cov==4.1.0 - # via -r requirements.in -python-dateutil==2.8.2 - # via - # jupyterhub - # jupyterhub-idle-culler - # kubernetes - # kubernetes-asyncio -python-json-logger==2.0.7 - # via jupyter-telemetry -python-slugify==8.0.1 - # via jupyterhub-kubespawner -pyyaml==6.0 - # via - # jupyterhub-kubespawner - # kubernetes - # kubernetes-asyncio -requests==2.31.0 - # via - # jupyterhub - # kubernetes - # oauthenticator - # requests-mock - # requests-oauthlib -requests-mock==1.10.0 - # via -r requirements.in -requests-oauthlib==1.3.1 - # via kubernetes -rsa==4.9 - # via google-auth -ruamel-yaml==0.17.31 - # via - # jupyter-telemetry - # oauthenticator -ruamel-yaml-clib==0.2.7 - # via ruamel-yaml -six==1.16.0 - # via - # google-auth - # kubernetes - # kubernetes-asyncio - # python-dateutil - # requests-mock -sqlalchemy==2.0.15 - # via - # alembic - # jupyterhub -text-unidecode==1.3 - # via python-slugify -tomli==2.0.1 - # via - # coverage - # pytest -tornado==6.3.2 - # via - # jupyterhub - # jupyterhub-idle-culler -traitlets==5.9.0 - # via - # jupyter-telemetry - # jupyterhub - # jupyterhub-kubespawner -typing-extensions==4.6.3 - # via - # alembic - # annotated-types - # pydantic - # pydantic-core - # sqlalchemy -urllib3==1.26.16 - # via - # google-auth - # jupyterhub-kubespawner - # kubernetes - # kubernetes-asyncio - # requests -websocket-client==1.5.2 - # via kubernetes -yarl==1.9.2 - # via aiohttp -zipp==3.17.0 - # via - # importlib-metadata - # importlib-resources - -# The following packages are considered to be unsafe in a requirements file: -# setuptools +kubernetes==31.0.0 +loguru==0.7.2 diff --git a/setup.cfg b/setup.cfg index 710cebb..03d2509 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,3 +1,3 @@ [metadata] name = application-context-hub -version = 1.2.0 +version = 1.3.0 diff --git a/sk-k8s/cluster-role-binding.yaml b/sk-k8s/cluster-role-binding.yaml new file mode 100644 index 0000000..e929f32 --- /dev/null +++ b/sk-k8s/cluster-role-binding.yaml @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: jupyter-rbac +subjects: + - kind: ServiceAccount + name: hub + namespace: jupyter +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cluster-admin diff --git a/sk-k8s/job.yaml b/sk-k8s/job.yaml new file mode 100644 index 0000000..c122f07 --- /dev/null +++ b/sk-k8s/job.yaml @@ -0,0 +1,26 @@ +apiVersion: batch/v1 +kind: Job + +metadata: + name: hub-content-init + namespace: jupyter +spec: + template: + spec: + containers: + - name: hub-content-init + image: ubuntu:22.04 + command: ["/bin/bash", "-c"] + args: ["/init/init.sh"] + volumeMounts: + - name: init-script-volume + mountPath: /init/init.sh + readOnly: true + subPath: init.sh + restartPolicy: Never + volumes: + - name: init-script-volume + configMap: + defaultMode: 0700 + name: init-script-configmap + backoffLimit: 4 diff --git a/sk-k8s/script.yaml b/sk-k8s/script.yaml new file mode 100644 index 0000000..0eb2a35 --- /dev/null +++ b/sk-k8s/script.yaml @@ -0,0 +1,22 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: init-script-configmap + namespace: jupyter +data: + init.sh: | + #!/bin/bash + apt update + apt install -y jq curl + + token=`curl -X POST -d '{"auth": {"username": "jovyan", "token": "12345"}}' http://application-hub-hub.jupyter.svc.cluster.local:8081/hub/api/users/jovyan/tokens | jq -r '.token'` + + curl --header "Authorization: Bearer $token" http://application-hub-hub.jupyter.svc.cluster.local:8081/hub/api/groups + + # create groups + for group in group-a group-b group-c + do + curl --request POST --location http://application-hub-hub.jupyter.svc.cluster.local:8081/hub/api/groups/${group} --header "Authorization: Bearer $token" --header 'Content-Type: application/json' + # add user to group + curl --request POST --location http://application-hub-hub.jupyter.svc.cluster.local:8081/hub/api/groups/${group}/users --header "Authorization: Bearer $token" --header 'Content-Type: application/json' --data '{"users": ["jovyan", "alice", "bob"]}' + done \ No newline at end of file diff --git a/skaffold.yaml b/skaffold.yaml new file mode 100644 index 0000000..54d1aed --- /dev/null +++ b/skaffold.yaml @@ -0,0 +1,47 @@ +apiVersion: skaffold/v4beta9 +kind: Config +build: + artifacts: + - image: hubimage + +profiles: + - name: minikube + deploy: + helm: + releases: + - name: jupyterhub + remoteChart: eoepca/application-hub + namespace: jupyter + version: 4.0.1 + createNamespace: true + valuesFiles: [] + setValueTemplates: + jupyterhub.hub.image.name: "{{.IMAGE_REPO_hubimage}}" + jupyterhub.hub.image.tag: "{{.IMAGE_TAG_hubimage}}@{{.IMAGE_DIGEST_hubimage}}" + setValues: + jupyterhub.hub.image.pullSecrets: [] + jupyterhub.hub.db.pvc.storageClassName: standard + jupyterhub.hub.extraEnv.STORAGE_CLASS: standard + jupyterhub.proxy.secretToken: "032d5bfe141a7eab86d57587879b33c5d168617cacb339823d7f47fe2933f880" + jupyterhub.hub.extraEnv.APP_HUB_NAMESPACE: jupyter + jupyterhub.hub.networkPolicy.enabled: false + setFiles: + configYml: ./files/hub/config.yml + jupyterConfig: ./files/hub/jupyterhub_config.py + hooks: + before: + - host: + command: ["sh", "-c", "config-generator/generate-config.sh"] + os: [darwin, linux] + manifests: + rawYaml: + - sk-k8s/cluster-role-binding.yaml + - sk-k8s/script.yaml + - sk-k8s/job.yaml + +portForward: +- resourceType: service + resourceName: application-hub-proxy-public + namespace: jupyter # Optional, if you are using a specific namespace + port: 80 # Target port on the pod + localPort: 8000 # Local port on your machine diff --git a/tests/tests_k8s.py b/tests/tests_k8s.py index 316f6a0..3b6fe57 100644 --- a/tests/tests_k8s.py +++ b/tests/tests_k8s.py @@ -45,7 +45,7 @@ class TestK8s(unittest.TestCase): def setUpClass(cls): os.environ["KUBECONFIG"] = "/home/mambauser/.kube/kubeconfig-t2-dev.yaml" - cls.app_hub_context = DefaulfApplicationHubContext( + cls.app_hub_context = DefaultApplicationHubContext( namespace="a_namespace", spawner=spawner, config_path="tests/data/config.yml", @@ -54,7 +54,7 @@ def setUpClass(cls): ) def test_obj(self): - self.assertIs(type(self.app_hub_context), DefaulfApplicationHubContext) + self.assertIs(type(self.app_hub_context), DefaultApplicationHubContext) def test_client(self): self.assertIsInstance(self.app_hub_context._get_core_v1_api(), client.CoreV1Api) diff --git a/validate_config.py b/validate_config.py deleted file mode 100644 index f320633..0000000 --- a/validate_config.py +++ /dev/null @@ -1,9 +0,0 @@ -from application_hub_context.parser import ConfigParser - -ws_config_parser = ConfigParser.read_file( - config_path="jupyterhub/files/hub/config.yml", user_groups=["group-2"] -) - -print(ws_config_parser.get_profile_by_slug(slug="ellip_studio_labs").dict()) - -print(ws_config_parser.get_profile_config_maps(profile_id="profile_studio_labs"))