diff --git a/backend/Dockerfile b/backend/Dockerfile index 07f8dac..5b1d6bb 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.13.3-slim +FROM python:3.13.9-slim WORKDIR /app diff --git a/backend/application_quality/settings.py b/backend/application_quality/settings.py index 8b29c56..0456e1a 100644 --- a/backend/application_quality/settings.py +++ b/backend/application_quality/settings.py @@ -12,6 +12,8 @@ import os from pathlib import Path +from urllib.parse import urlparse + # Build paths inside the project like this: BASE_DIR / 'subdir'. BASE_DIR = Path(__file__).resolve().parent.parent @@ -54,7 +56,23 @@ }, } -ALLOWED_HOSTS = ["*"] +# Build the ALLOWED_HOSTS variable + +# PUBLIC_URL should contain a scheme, and possibly a port number +allowed_host = os.getenv("PUBLIC_URL") +if "://" not in allowed_host: + allowed_host = "http://" + allowed_host +parsed_allowed_host = urlparse(allowed_host) +allowed_domain = parsed_allowed_host.netloc +if ":" in allowed_domain: + # Remove the port number, if any + allowed_domain = allowed_domain.split(":")[0] + +ALLOWED_HOSTS = [allowed_domain] +add_hosts_raw = os.getenv("ADDITIONAL_ALLOWED_HOSTS") +if add_hosts_raw: + add_hosts_list = [host.strip() for host in add_hosts_raw.split(',') if host.strip()] + ALLOWED_HOSTS.extend(add_hosts_list) # Application definition @@ -71,6 +89,7 @@ "corsheaders", "backend", "mozilla_django_oidc", + "django_svelte_jsoneditor", # See SVELTE_JSONEDITOR_PROPS, below ] MIDDLEWARE = [ @@ -150,6 +169,25 @@ ] +# svelte-jsoneditor default properties for all widgets rendered in the app. +# See: https://django-svelte-jsoneditor.readthedocs.io/en/latest/settings.html + +SVELTE_JSONEDITOR_PROPS = { + # Just the subset to customize: + "mode": "tree", + "mainMenuBar": True, + "navigationBar": True, + "statusBar": True, + "askToFormat": True, + "readOnly": False, + "indentation": 4, + "tabSize": 4, + "escapeControlCharacters": False, + "escapeUnicodeCharacters": False, + "flattenColumns": True, +} + + # Internationalization # https://docs.djangoproject.com/en/5.1/topics/i18n/ @@ -225,9 +263,8 @@ CORS_ORIGIN_ALLOW_ALL = True -CORS_ALLOWED_ORIGINS = [os.getenv("PUBLIC_URL")] # Frontend URL - CORS_ALLOW_HEADERS = "*" +CORS_ALLOWED_ORIGINS = [os.getenv("PUBLIC_URL")] # Frontend URL #CSRF_COOKIE_SECURE = False #CSRF_USE_SESSIONS = True diff --git a/backend/backend/admin.py b/backend/backend/admin.py index 35b29d3..032b90d 100644 --- a/backend/backend/admin.py +++ b/backend/backend/admin.py @@ -1,20 +1,42 @@ -from django.contrib import admin from backend import models +from django.contrib import admin +from django.db.models import JSONField +# https://django-svelte-jsoneditor.readthedocs.io/en/latest/index.html +# Customised using SVELTE_JSONEDITOR_PROPS in the Django settings +from django_svelte_jsoneditor.widgets import SvelteJSONEditorWidget + + + class PipelineSettings(admin.ModelAdmin): list_display = ("name", "description", "owner", "version", "created_at", "edited_at") + formfield_overrides = { + JSONField: {"widget": SvelteJSONEditorWidget} + } class PipelineRunSettings(admin.ModelAdmin): list_display = ("__str__", "pipeline", "start_time", "completion_time", "status", "user", "started_by") + formfield_overrides = { + JSONField: {"widget": SvelteJSONEditorWidget} + } class ToolSettings(admin.ModelAdmin): - list_display = ("slug", "name", "version") + list_display = ("slug", "name", "version", "status", "available") + formfield_overrides = { + JSONField: {"widget": SvelteJSONEditorWidget} + } class CommandSettings(admin.ModelAdmin): list_display = ("slug", "name", "version") + formfield_overrides = { + JSONField: {"widget": SvelteJSONEditorWidget} + } class JobReportSettings(admin.ModelAdmin): list_display = ("__str__", "name", "created_at") + formfield_overrides = { + JSONField: {"widget": SvelteJSONEditorWidget} + } admin.site.register(models.Pipeline, PipelineSettings) admin.site.register(models.PipelineRun, PipelineRunSettings) diff --git a/backend/backend/fixtures/2025-04-03.json b/backend/backend/fixtures/2025-04-03.json deleted file mode 100644 index 693da7d..0000000 --- a/backend/backend/fixtures/2025-04-03.json +++ /dev/null @@ -1,634 +0,0 @@ -[ - { - "model": "backend.pipeline", - "pk": 6, - "fields": { - "name": "Python pipeline", - "description": "Runs a series of static analysis tools on python files", - "template": "#!/usr/bin/env cwltool\r\n\r\n$graph:\r\n- class: Workflow\r\n\r\n requirements:\r\n SubworkflowFeatureRequirement: {}\r\n\r\n inputs:\r\n{%- for tool in subworkflows %}\r\n {%- for tool_name, params in tool.user_params.items() %}\r\n {%- for param_name, param in params.items() %}\r\n {{ tool.slug }}.{{ tool_name }}.{{ param_name }}:\r\n label: \"{{ param.label or param_name }}\"\r\n {%- if param.doc %}\r\n doc: |-\r\n {{ param.doc }}\r\n {%- endif %}\r\n type: {{ param.type }}\r\n default: {{ param.default | tojson }}\r\n {%- endfor %}\r\n {%- endfor %}\r\n{%- endfor %}\r\n branch:\r\n type: string\r\n default: ''\r\n pipeline_id: string\r\n run_id: string\r\n server_url: string\r\n\r\n outputs: []\r\n\r\n steps:\r\n {% for tool in subworkflows %}{{ tool.pipeline_step | indent(4) }}\r\n {% endfor %}\r\n id: main\r\n{% for tool in subworkflows %}\r\n{{ tool.definition }}\r\n{% endfor %}\r\ncwlVersion: v1.0", - "default_inputs": {}, - "owner": null, - "created_at": null, - "edited_at": null, - "version": "0.1", - "tools": [ - "bandit_subworkflow", - "clone_subworkflow", - "flake8_subworkflow", - "pylint_subworkflow", - "ruff_subworkflow" - ] - } - }, - { - "model": "backend.pipeline", - "pk": 8, - "fields": { - "name": "Notebook pipeline", - "description": "Runs a static analysis and then executes Jupyter Notebooks files (ipynb).", - "template": "#!/usr/bin/env cwltool\r\n\r\n$graph:\r\n- class: Workflow\r\n\r\n requirements:\r\n SubworkflowFeatureRequirement: {}\r\n\r\n inputs:\r\n{%- for tool in subworkflows %}\r\n {%- for tool_name, params in tool.user_params.items() %}\r\n {%- for param_name, param in params.items() %}\r\n {{ tool.slug }}.{{ tool_name }}.{{ param_name }}:\r\n label: \"{{ param.label or param_name }}\"\r\n {%- if param.doc %}\r\n doc: |-\r\n {{ param.doc }}\r\n {%- endif %}\r\n type: {{ param.type }}\r\n default: {{ param.default | tojson }}\r\n {%- endfor %}\r\n {%- endfor %}\r\n{%- endfor %}\r\n branch:\r\n type: string\r\n default: ''\r\n pipeline_id: string\r\n run_id: string\r\n server_url: string\r\n\r\n outputs: []\r\n\r\n steps:\r\n {% for tool in subworkflows %}{{ tool.pipeline_step | indent(4) }}\r\n {% endfor %}\r\n id: main\r\n{% for tool in subworkflows %}\r\n{{ tool.definition }}\r\n{% endfor %}\r\ncwlVersion: v1.0", - "default_inputs": {}, - "owner": null, - "created_at": null, - "edited_at": null, - "version": "0.1", - "tools": [ - "clone_subworkflow", - "ipynb_specs_checker_subworkflow", - "papermill_subworkflow", - "ruff_ipynb_subworkflow" - ] - } - }, - { - "model": "backend.pipeline", - "pk": 11, - "fields": { - "name": "Docker pipeline", - "description": "Finds vulnerabilities and misconfigurations in Docker images.", - "template": "#!/usr/bin/env cwltool\r\n\r\n$graph:\r\n- class: Workflow\r\n\r\n requirements:\r\n SubworkflowFeatureRequirement: {}\r\n\r\n inputs:\r\n{%- for tool in subworkflows %}\r\n {%- for tool_name, params in tool.user_params.items() %}\r\n {%- for param_name, param in params.items() %}\r\n {{ tool.slug }}.{{ tool_name }}.{{ param_name }}:\r\n label: \"{{ param.label or param_name }}\"\r\n {%- if param.doc %}\r\n doc: |-\r\n {{ param.doc }}\r\n {%- endif %}\r\n type: {{ param.type }}\r\n default: {{ param.default | tojson }}\r\n {%- endfor %}\r\n {%- endfor %}\r\n{%- endfor %}\r\n pipeline_id: string\r\n run_id: string\r\n server_url: string\r\n\r\n outputs: []\r\n\r\n steps:\r\n {% for tool in subworkflows %}{{ tool.pipeline_step | indent(4) }}\r\n {% endfor %}\r\n id: main\r\n{% for tool in subworkflows %}\r\n{{ tool.definition }}\r\n{% endfor %}\r\ncwlVersion: v1.0", - "default_inputs": {}, - "owner": null, - "created_at": null, - "edited_at": null, - "version": "0.1", - "tools": [ - "trivy_subworkflow" - ] - } - }, - { - "model": "backend.tag", - "pk": 1, - "fields": { - "name": "asset: python" - } - }, - { - "model": "backend.tag", - "pk": 2, - "fields": { - "name": "asset: other" - } - }, - { - "model": "backend.tag", - "pk": 3, - "fields": { - "name": "asset: cwl" - } - }, - { - "model": "backend.tag", - "pk": 4, - "fields": { - "name": "asset: notebook" - } - }, - { - "model": "backend.tag", - "pk": 5, - "fields": { - "name": "type: best practice" - } - }, - { - "model": "backend.tag", - "pk": 6, - "fields": { - "name": "type: app quality" - } - }, - { - "model": "backend.tag", - "pk": 7, - "fields": { - "name": "type: app performance" - } - }, - { - "model": "backend.tag", - "pk": 8, - "fields": { - "name": "type: init" - } - }, - { - "model": "backend.tag", - "pk": 9, - "fields": { - "name": "asset: docker" - } - }, - { - "model": "backend.subworkflow", - "pk": "ap_validator_subworkflow", - "fields": { - "name": "Application Package Validator", - "description": "Validation tool for checking OGC compliance of CWL files for application packages.", - "pipeline_step": "ap_validator_subworkflow:\r\n in:\r\n ap_validator.detail: ap_validator_subworkflow.ap_validator.detail\r\n ap_validator.entry_point: ap_validator_subworkflow.ap_validator.entry_point\r\n filter.regex: ap_validator_subworkflow.filter.regex\r\n pipeline_id: pipeline_id\r\n run_id: run_id\r\n server_url: server_url\r\n source_directory: clone_step/repo_directory\r\n run: '#ap_validator_subworkflow'\r\n out: []", - "definition": "- class: Workflow\r\n\r\n requirements:\r\n ScatterFeatureRequirement: {}\r\n\r\n inputs:\r\n name:\r\n type: string\r\n default: ap_validator\r\n ap_validator.detail: string\r\n ap_validator.entry_point: string\r\n filter.regex: string\r\n pipeline_id: string\r\n run_id: string\r\n server_url: string\r\n source_directory: Directory\r\n\r\n outputs:\r\n []\r\n\r\n steps:\r\n filter_ap_validator_step:\r\n in:\r\n regex: filter.regex\r\n source_directory: source_directory\r\n run: '#filter_tool'\r\n out:\r\n - file_list\r\n ap_validator_step:\r\n in:\r\n file_path: filter_ap_validator_step/file_list\r\n source_directory: source_directory\r\n detail: ap_validator.detail\r\n entry_point: ap_validator.entry_point\r\n scatter: file_path\r\n run: '#ap_validator_tool'\r\n out:\r\n - ap_validator_report\r\n save_ap_validator_step:\r\n in:\r\n name: name\r\n pipeline_id: pipeline_id\r\n report: ap_validator_step/ap_validator_report\r\n run_id: run_id\r\n server_url: server_url\r\n scatter: report\r\n run: '#save_tool'\r\n out: []\r\n id: ap_validator_subworkflow\r\n{% for tool in tools %}{{ tool.definition }}\r\n{% endfor %}", - "user_params": { - "filter": { - "regex": { - "type": "string", - "label": "regex", - "default": ".*\\.cwl" - } - }, - "ap_validator": { - "detail": { - "doc": "Output detail (none|errors|hints|all). Default: hints", - "type": "string", - "label": "Detail", - "default": "hints" - }, - "entry_point": { - "doc": "Name of entry point (Workflow or CommandLineTool)", - "type": "string", - "label": "Entry point", - "default": "main" - } - } - }, - "version": "0.1", - "tags": [ - 3, - 5 - ], - "tools": [ - "ap_validator", - "filter", - "save" - ] - } - }, - { - "model": "backend.subworkflow", - "pk": "bandit_subworkflow", - "fields": { - "name": "Bandit", - "description": "Bandit - Bandit is a tool designed to find common security issues in Python code", - "pipeline_step": "bandit_subworkflow:\r\n in:\r\n bandit.verbose: bandit_subworkflow.bandit.verbose\r\n filter.regex: bandit_subworkflow.filter.regex\r\n pipeline_id: pipeline_id\r\n run_id: run_id\r\n server_url: server_url\r\n source_directory: clone_step/repo_directory\r\n run: '#bandit_subworkflow'\r\n out: []", - "definition": "- class: Workflow\r\n\r\n inputs:\r\n name:\r\n type: string\r\n default: bandit\r\n bandit.verbose: boolean\r\n filter.regex: string\r\n pipeline_id: string\r\n run_id: string\r\n server_url: string\r\n source_directory: Directory\r\n\r\n outputs:\r\n bandit_report:\r\n type: File\r\n outputSource: bandit_step/bandit_report\r\n\r\n steps:\r\n bandit_step:\r\n in:\r\n file_list: filter_bandit_step/file_list\r\n source_directory: source_directory\r\n verbose: bandit.verbose\r\n run: '#bandit_tool'\r\n out:\r\n - bandit_report\r\n filter_bandit_step:\r\n in:\r\n regex: filter.regex\r\n source_directory: source_directory\r\n run: '#filter_tool'\r\n out:\r\n - file_list\r\n save_bandit_step:\r\n in:\r\n name: name\r\n pipeline_id: pipeline_id\r\n report: bandit_step/bandit_report\r\n run_id: run_id\r\n server_url: server_url\r\n run: '#save_tool'\r\n out: []\r\n id: bandit_subworkflow\r\n{% for tool in tools %}{{ tool.definition }}\r\n{% endfor %}", - "user_params": { - "bandit": { - "verbose": { - "doc": "Output extra information like excluded and included files.", - "type": "boolean", - "label": "Verbose", - "default": false - } - }, - "filter": { - "regex": { - "type": "string", - "label": "regex", - "default": ".*\\.py" - } - } - }, - "version": "0.1", - "tags": [ - 1, - 6 - ], - "tools": [ - "bandit", - "filter", - "save" - ] - } - }, - { - "model": "backend.subworkflow", - "pk": "clone_subworkflow", - "fields": { - "name": "Clone repo", - "description": "git-clone - Clone a repository into a new directory", - "pipeline_step": "clone_step:\r\n in:\r\n clone.repo_branch: clone_subworkflow.clone.repo_branch\r\n clone.repo_url: clone_subworkflow.clone.repo_url\r\n run: '#clone_subworkflow'\r\n out:\r\n - repo_directory", - "definition": "- class: Workflow\r\n\r\n inputs:\r\n clone.repo_branch:\r\n type: string\r\n default: ''\r\n clone.repo_url: string\r\n\r\n outputs:\r\n repo_directory:\r\n type: Directory\r\n outputSource: clone_tool_step/repo_directory\r\n\r\n steps:\r\n clone_tool_step:\r\n in:\r\n repo_branch: clone.repo_branch\r\n repo_url: clone.repo_url\r\n run: '#clone_tool'\r\n out:\r\n - repo_directory\r\n id: clone_subworkflow\r\n{% for tool in tools %}{{ tool.definition }}\r\n{% endfor %}", - "user_params": { - "clone": { - "repo_url": { - "doc": "URL to the repository to clone.", - "type": "string", - "label": "Repo URL", - "default": "" - }, - "repo_branch": { - "doc": "Branch to checkout instead of the remote HEAD.", - "type": "string", - "label": "Repo branch", - "default": "" - } - } - }, - "version": "0.1", - "tags": [ - 2, - 8 - ], - "tools": [ - "clone" - ] - } - }, - { - "model": "backend.subworkflow", - "pk": "flake8_subworkflow", - "fields": { - "name": "Flake8", - "description": "flake8 - Style guide enforcement tool for Python", - "pipeline_step": "flake8_subworkflow:\r\n in:\r\n filter.regex: flake8_subworkflow.filter.regex\r\n flake8.verbose: flake8_subworkflow.flake8.verbose\r\n pipeline_id: pipeline_id\r\n run_id: run_id\r\n server_url: server_url\r\n source_directory: clone_step/repo_directory\r\n run: '#flake8_subworkflow'\r\n out: []", - "definition": "- class: Workflow\r\n\r\n inputs:\r\n name:\r\n type: string\r\n default: flake8\r\n filter.regex: string\r\n flake8.verbose: boolean\r\n pipeline_id: string\r\n run_id: string\r\n server_url: string\r\n source_directory: Directory\r\n\r\n outputs:\r\n flake8_report:\r\n type: File\r\n outputSource: flake8_step/flake8_report\r\n\r\n steps:\r\n filter_flake8_step:\r\n in:\r\n regex: filter.regex\r\n source_directory: source_directory\r\n run: '#filter_tool'\r\n out:\r\n - file_list\r\n flake8_step:\r\n in:\r\n file_list: filter_flake8_step/file_list\r\n source_directory: source_directory\r\n verbose: flake8.verbose\r\n run: '#flake8_tool'\r\n out:\r\n - flake8_report\r\n save_flake8_step:\r\n in:\r\n name: name\r\n pipeline_id: pipeline_id\r\n report: flake8_step/flake8_report\r\n run_id: run_id\r\n server_url: server_url\r\n run: '#save_tool'\r\n out: []\r\n id: flake8_subworkflow\r\n{% for tool in tools %}{{ tool.definition }}\r\n{% endfor %}", - "user_params": { - "filter": { - "regex": { - "type": "string", - "label": "regex", - "default": ".*\\.py" - } - }, - "flake8": { - "verbose": { - "doc": "Increase the verbosity of Flake8’s output.", - "type": "boolean", - "label": "Verbose", - "default": false - } - } - }, - "version": "0.1", - "tags": [ - 1, - 5 - ], - "tools": [ - "filter", - "flake8", - "save" - ] - } - }, - { - "model": "backend.subworkflow", - "pk": "ipynb_specs_checker_subworkflow", - "fields": { - "name": "Jupyter Notebook Best Practices Checker", - "description": "Best practices checker for Jupyter notebook specs", - "pipeline_step": "ipynb_specs_checker_subworkflow:\r\n in:\r\n filter.regex: ipynb_specs_checker_subworkflow.filter.regex\r\n pipeline_id: pipeline_id\r\n ipynb_specs_checker.schema: ipynb_specs_checker_subworkflow.ipynb_specs_checker.schema\r\n run_id: run_id\r\n server_url: server_url\r\n source_directory: clone_step/repo_directory\r\n run: '#ipynb_specs_checker_subworkflow'\r\n out: []", - "definition": "- class: Workflow\r\n\r\n inputs:\r\n name:\r\n type: string\r\n default: ipynb_specs_checker\r\n filter.regex: string\r\n pipeline_id: string\r\n ipynb_specs_checker.schema: string\r\n run_id: string\r\n server_url: string\r\n source_directory: Directory\r\n\r\n outputs:\r\n ipynb_specs_checker_report:\r\n type: File\r\n outputSource: ipynb_specs_checker_step/ipynb_specs_checker_report\r\n\r\n steps:\r\n filter_ipynb_specs_checker_step:\r\n in:\r\n regex: filter.regex\r\n source_directory: source_directory\r\n run: '#filter_tool'\r\n out:\r\n - file_list\r\n ipynb_specs_checker_step:\r\n in:\r\n file_list: filter_ipynb_specs_checker_step/file_list\r\n source_directory: source_directory\r\n schema: ipynb_specs_checker.schema\r\n run: '#ipynb_specs_checker_tool'\r\n out:\r\n - ipynb_specs_checker_report\r\n save_ipynb_specs_checker_step:\r\n in:\r\n name: name\r\n pipeline_id: pipeline_id\r\n report: ipynb_specs_checker_step/ipynb_specs_checker_report\r\n run_id: run_id\r\n server_url: server_url\r\n run: '#save_tool'\r\n out: []\r\n id: ipynb_specs_checker_subworkflow\r\n{% for tool in tools %}{{ tool.definition }}\r\n{% endfor %}", - "user_params": { - "filter": { - "regex": { - "type": "string", - "label": "regex", - "default": ".*\\.ipynb" - } - }, - "ipynb_specs_checker": { - "schema": { - "type": "string", - "default": "eumetsat" - } - } - }, - "version": "0.1", - "tags": [ - 4, - 5 - ], - "tools": [ - "filter", - "ipynb_specs_checker", - "save" - ] - } - }, - { - "model": "backend.subworkflow", - "pk": "papermill_subworkflow", - "fields": { - "name": "Papermill", - "description": "Papermill is a tool for parameterizing and executing Jupyter Notebooks.", - "pipeline_step": "papermill_subworkflow:\r\n in:\r\n filter.regex: papermill_subworkflow.filter.regex\r\n pipeline_id: pipeline_id\r\n run_id: run_id\r\n server_url: server_url\r\n source_directory: clone_step/repo_directory\r\n run: '#papermill_subworkflow'\r\n out: []", - "definition": "- class: Workflow\r\n\r\n requirements:\r\n ScatterFeatureRequirement: {}\r\n\r\n inputs:\r\n name:\r\n type: string\r\n default: papermill\r\n filter.regex: string\r\n pipeline_id: string\r\n run_id: string\r\n server_url: string\r\n source_directory: Directory\r\n\r\n outputs:\r\n []\r\n\r\n steps:\r\n filter_papermill_step:\r\n in:\r\n regex: filter.regex\r\n source_directory: source_directory\r\n run: '#filter_tool'\r\n out:\r\n - file_list\r\n papermill_step:\r\n in:\r\n notebook_path: filter_papermill_step/file_list\r\n source_directory: source_directory\r\n scatter: notebook_path\r\n run: '#papermill_tool'\r\n out:\r\n - output_nb\r\n save_papermill_step:\r\n in:\r\n name: name\r\n pipeline_id: pipeline_id\r\n report: papermill_step/output_nb\r\n run_id: run_id\r\n server_url: server_url\r\n scatter: report\r\n run: '#save_tool'\r\n out: []\r\n id: papermill_subworkflow\r\n{% for tool in tools %}{{ tool.definition }}\r\n{% endfor %}", - "user_params": { - "filter": { - "regex": { - "type": "string", - "label": "regex", - "default": ".*\\.ipynb" - } - } - }, - "version": "0.1", - "tags": [ - 4, - 7 - ], - "tools": [ - "filter", - "papermill", - "save" - ] - } - }, - { - "model": "backend.subworkflow", - "pk": "pylint_subworkflow", - "fields": { - "name": "Pylint", - "description": "pylint - Static code analyser tool for Python", - "pipeline_step": "pylint_subworkflow:\r\n in:\r\n filter.regex: pylint_subworkflow.filter.regex\r\n pipeline_id: pipeline_id\r\n pylint.disable: pylint_subworkflow.pylint.disable\r\n pylint.errors_only: pylint_subworkflow.pylint.errors_only\r\n pylint.verbose: pylint_subworkflow.pylint.verbose\r\n run_id: run_id\r\n server_url: server_url\r\n source_directory: clone_step/repo_directory\r\n run: '#pylint_subworkflow'\r\n out: []", - "definition": "- class: Workflow\r\n\r\n inputs:\r\n name:\r\n type: string\r\n default: pylint\r\n filter.regex: string\r\n pipeline_id: string\r\n pylint.disable: string\r\n pylint.errors_only: boolean\r\n pylint.verbose: boolean\r\n run_id: string\r\n server_url: string\r\n source_directory: Directory\r\n\r\n outputs:\r\n pylint_report:\r\n type: File\r\n outputSource: pylint_step/pylint_report\r\n\r\n steps:\r\n filter_pylint_step:\r\n in:\r\n regex: filter.regex\r\n source_directory: source_directory\r\n run: '#filter_tool'\r\n out:\r\n - file_list\r\n pylint_step:\r\n in:\r\n disable: pylint.disable\r\n errors_only: pylint.errors_only\r\n file_list: filter_pylint_step/file_list\r\n source_directory: source_directory\r\n verbose: pylint.verbose\r\n run: '#pylint_tool'\r\n out:\r\n - pylint_report\r\n save_pylint_step:\r\n in:\r\n name: name\r\n pipeline_id: pipeline_id\r\n report: pylint_step/pylint_report\r\n run_id: run_id\r\n server_url: server_url\r\n run: '#save_tool'\r\n out: []\r\n id: pylint_subworkflow\r\n{% for tool in tools %}{{ tool.definition }}\r\n{% endfor %}", - "user_params": { - "filter": { - "regex": { - "type": "string", - "label": "regex", - "default": ".*\\.py" - } - }, - "pylint": { - "disable": { - "doc": "Disable the message, report, category or checker with the given id(s).", - "type": "string", - "label": "Disable IDs", - "default": "E0401" - }, - "verbose": { - "doc": "In verbose mode, extra non-checker-related info will be displayed.", - "type": "boolean", - "label": "Verbose", - "default": false - }, - "errors_only": { - "doc": "In error mode, messages with a category besides ERROR or FATAL are suppressed, and no reports are done by default. Error mode is compatible with disabling specific errors.", - "type": "boolean", - "label": "Errors only", - "default": false - } - } - }, - "version": "0.1", - "tags": [ - 1, - 5 - ], - "tools": [ - "filter", - "pylint", - "save" - ] - } - }, - { - "model": "backend.subworkflow", - "pk": "ruff_ipynb_subworkflow", - "fields": { - "name": "Ruff - Notebook", - "description": "Ruff - An extremely fast Python linter and code formatter, written in Rust", - "pipeline_step": "ruff_ipynb_subworkflow:\r\n in:\r\n filter.regex: ruff_ipynb_subworkflow.filter.regex\r\n pipeline_id: pipeline_id\r\n ruff.verbose: ruff_ipynb_subworkflow.ruff.verbose\r\n run_id: run_id\r\n server_url: server_url\r\n source_directory: clone_step/repo_directory\r\n run: '#ruff_ipynb_subworkflow'\r\n out: []", - "definition": "- class: Workflow\r\n\r\n inputs:\r\n name:\r\n type: string\r\n default: ruff\r\n filter.regex: string\r\n pipeline_id: string\r\n ruff.verbose: boolean\r\n run_id: string\r\n server_url: string\r\n source_directory: Directory\r\n\r\n outputs:\r\n ruff_report:\r\n type: File\r\n outputSource: ruff_step/ruff_report\r\n\r\n steps:\r\n filter_ruff_step:\r\n in:\r\n regex: filter.regex\r\n source_directory: source_directory\r\n run: '#filter_tool'\r\n out:\r\n - file_list\r\n ruff_step:\r\n in:\r\n file_list: filter_ruff_step/file_list\r\n source_directory: source_directory\r\n verbose: ruff.verbose\r\n run: '#ruff_tool'\r\n out:\r\n - ruff_report\r\n save_ruff_step:\r\n in:\r\n name: name\r\n pipeline_id: pipeline_id\r\n report: ruff_step/ruff_report\r\n run_id: run_id\r\n server_url: server_url\r\n run: '#save_tool'\r\n out: []\r\n id: ruff_ipynb_subworkflow\r\n{% for tool in tools %}{{ tool.definition }}\r\n{% endfor %}", - "user_params": { - "ruff": { - "verbose": { - "doc": "Enable verbose logging.", - "type": "boolean", - "label": "Verbose", - "default": false - } - }, - "filter": { - "regex": { - "type": "string", - "label": "regex", - "default": ".*\\.ipynb" - } - } - }, - "version": "0.1", - "tags": [ - 4, - 5 - ], - "tools": [ - "filter", - "ruff", - "save" - ] - } - }, - { - "model": "backend.subworkflow", - "pk": "ruff_subworkflow", - "fields": { - "name": "Ruff", - "description": "Ruff - An extremely fast Python linter and code formatter, written in Rust", - "pipeline_step": "ruff_subworkflow:\r\n in:\r\n filter.regex: ruff_subworkflow.filter.regex\r\n pipeline_id: pipeline_id\r\n ruff.verbose: ruff_subworkflow.ruff.verbose\r\n run_id: run_id\r\n server_url: server_url\r\n source_directory: clone_step/repo_directory\r\n run: '#ruff_subworkflow'\r\n out: []", - "definition": "- class: Workflow\r\n\r\n inputs:\r\n name:\r\n type: string\r\n default: ruff\r\n filter.regex: string\r\n pipeline_id: string\r\n ruff.verbose: boolean\r\n run_id: string\r\n server_url: string\r\n source_directory: Directory\r\n\r\n outputs:\r\n ruff_report:\r\n type: File\r\n outputSource: ruff_step/ruff_report\r\n\r\n steps:\r\n filter_ruff_step:\r\n in:\r\n regex: filter.regex\r\n source_directory: source_directory\r\n run: '#filter_tool'\r\n out:\r\n - file_list\r\n ruff_step:\r\n in:\r\n file_list: filter_ruff_step/file_list\r\n source_directory: source_directory\r\n verbose: ruff.verbose\r\n run: '#ruff_tool'\r\n out:\r\n - ruff_report\r\n save_ruff_step:\r\n in:\r\n name: name\r\n pipeline_id: pipeline_id\r\n report: ruff_step/ruff_report\r\n run_id: run_id\r\n server_url: server_url\r\n run: '#save_tool'\r\n out: []\r\n id: ruff_subworkflow\r\n{% for tool in tools %}{{ tool.definition }}\r\n{% endfor %}", - "user_params": { - "ruff": { - "verbose": { - "doc": "Enable verbose logging.", - "type": "boolean", - "label": "Verbose", - "default": false - } - }, - "filter": { - "regex": { - "type": "string", - "label": "regex", - "default": ".*\\.py" - } - } - }, - "version": "0.1", - "tags": [ - 1, - 5 - ], - "tools": [ - "filter", - "ruff", - "save" - ] - } - }, - { - "model": "backend.subworkflow", - "pk": "sonarqube", - "fields": { - "name": "SonarQube", - "description": "SonarQube - Code Quality, Security & Static Analysis Tool\r\n\r\nThis tool creates a project in our internal SonarQube server, sends it the code for analysis, and then retrieves the analysis results for storage in the database.", - "pipeline_step": "sonarqube_workflow_step:\r\n in:\r\n pipeline_id: pipeline_id\r\n repo_path: clone_step/repo_directory\r\n run_id: run_id\r\n server_url: server_url\r\n sonarqube_project_key: sonarqube_project_key\r\n sonarqube_project_name: sonarqube_project_name\r\n sonarqube_server: sonarqube_server\r\n sonarqube_token: sonarqube_token\r\n run: '#sonarqube_workflow'\r\n out: []", - "definition": "- class: Workflow\r\n\r\n inputs:\r\n name:\r\n type: string\r\n default: sonarqube\r\n pipeline_id:\r\n type: string\r\n repo_path:\r\n type: Directory\r\n run_id:\r\n type: string\r\n server_url:\r\n type: string\r\n sonarqube_project_key:\r\n type: string\r\n sonarqube_project_name:\r\n type: string\r\n sonarqube_server:\r\n type: string\r\n default: sonarqube-sonarqube.sonarqube:9000\r\n sonarqube_token:\r\n type: string\r\n\r\n outputs: []\r\n\r\n steps:\r\n save_sonarqube_step:\r\n in:\r\n name: name\r\n pipeline_id: pipeline_id\r\n report: sonarqube_get_report_step/sonarqube_report\r\n run_id: run_id\r\n server_url: server_url\r\n run: '#save_tool'\r\n out: []\r\n sonarqube_create_project_step:\r\n in:\r\n sonarqube_project_key: sonarqube_project_key\r\n sonarqube_project_name: sonarqube_project_name\r\n sonarqube_server: sonarqube_server\r\n sonarqube_token: sonarqube_token\r\n run: '#sonarqube_create_project_tool'\r\n out:\r\n - sonarqube_project_key\r\n - sonarqube_server\r\n - sonarqube_token\r\n sonarqube_get_report_step:\r\n in:\r\n sonarqube_project_key: sonarqube_scan_step/sonarqube_project_key\r\n sonarqube_server: sonarqube_scan_step/sonarqube_server\r\n sonarqube_token: sonarqube_scan_step/sonarqube_token\r\n run: '#sonarqube_get_report_tool'\r\n out:\r\n - sonarqube_project_key\r\n - sonarqube_server\r\n - sonarqube_token\r\n - sonarqube_report\r\n sonarqube_scan_step:\r\n in:\r\n sonarqube_project_key: sonarqube_create_project_step/sonarqube_project_key\r\n sonarqube_server: sonarqube_create_project_step/sonarqube_server\r\n sonarqube_token: sonarqube_create_project_step/sonarqube_token\r\n source_directory: repo_path\r\n run: '#sonarqube_scan_tool'\r\n out:\r\n - sonarqube_project_key\r\n - sonarqube_server\r\n - sonarqube_token\r\n id: sonarqube_workflow\r\n{% for tool in tools %}{{ tool.definition }}\r\n{% endfor %}", - "user_params": {}, - "version": "0.1", - "tags": [], - "tools": [ - "save", - "sonarqube_create_project", - "sonarqube_get_report", - "sonarqube_scan" - ] - } - }, - { - "model": "backend.subworkflow", - "pk": "trivy_subworkflow", - "fields": { - "name": "Trivy", - "description": "The all-in-one open source security scanner\r\nUse Trivy to find vulnerabilities (CVE) & misconfigurations (IaC) across code repositories, binary artifacts, container images, Kubernetes clusters, and more.", - "pipeline_step": "trivy_subworkflow:\r\n in:\r\n pipeline_id: pipeline_id\r\n run_id: run_id\r\n server_url: server_url\r\n trivy.image: trivy_subworkflow.trivy.image\r\n run: '#trivy_subworkflow'\r\n out: []", - "definition": "- class: Workflow\r\n\r\n inputs:\r\n name:\r\n type: string\r\n default: trivy\r\n pipeline_id: string\r\n run_id: string\r\n server_url: string\r\n trivy.image: string\r\n\r\n outputs: []\r\n\r\n steps:\r\n save_step:\r\n in:\r\n name: name\r\n pipeline_id: pipeline_id\r\n report: trivy_step/trivy_report\r\n run_id: run_id\r\n server_url: server_url\r\n run: '#save_tool'\r\n out: []\r\n trivy_step:\r\n in:\r\n image: trivy.image\r\n run: '#trivy_tool'\r\n out:\r\n - trivy_report\r\n id: trivy_subworkflow\r\n{% for tool in tools %}{{ tool.definition }}\r\n{% endfor %}", - "user_params": { - "trivy": { - "image": { - "type": "string", - "label": "Docker image", - "default": "alpine/git" - } - } - }, - "version": "0.1", - "tags": [ - 6, - 9 - ], - "tools": [ - "save", - "trivy" - ] - } - }, - { - "model": "backend.commandlinetool", - "pk": "ap_validator", - "fields": { - "name": "Application Package Validator", - "definition": "- class: CommandLineTool\r\n\r\n requirements:\r\n DockerRequirement:\r\n dockerPull: nexus.spaceapplications.com/repository/docker-eoepca/ap_validator:2025-03-05.1\r\n InitialWorkDirRequirement:\r\n listing:\r\n - entryname: script.sh\r\n entry: |-\r\n cd $(inputs.source_directory.path)\r\n\r\n ap-validator \\\\\r\n --format json \\\\\r\n --detail '$(inputs.detail)' \\\\\r\n --entry-point '$(inputs.entry_point)' \\\\\r\n '$(inputs.file_path)' > ~/ap_validator_report.json\r\n\r\n exit 0\r\n InlineJavascriptRequirement: {}\r\n\r\n inputs:\r\n file_path: string\r\n source_directory: Directory\r\n detail: string\r\n entry_point: string\r\n\r\n outputs:\r\n ap_validator_report:\r\n type: File\r\n outputBinding:\r\n glob: ap_validator_report.json\r\n\r\n baseCommand: sh\r\n arguments:\r\n - script.sh\r\n id: ap_validator_tool", - "version": "0.1" - } - }, - { - "model": "backend.commandlinetool", - "pk": "bandit", - "fields": { - "name": "Bandit", - "definition": "- class: CommandLineTool\r\n\r\n requirements:\r\n DockerRequirement:\r\n dockerPull: cytopia/bandit\r\n InitialWorkDirRequirement:\r\n listing:\r\n - entryname: script.sh\r\n entry: |-\r\n cd $(inputs.source_directory.path)\r\n\r\n PARAMS=\"-f $(inputs.output_format) -o $HOME/$(inputs.output_file)\"\r\n if [ \"$(inputs.exit_zero)\" == \"true\" ] ; then\r\n PARAMS=\"$PARAMS --exit-zero\"\r\n fi\r\n if [ \"$(inputs.verbose)\" == \"true\" ] ; then\r\n PARAMS=\"$PARAMS -v\"\r\n fi\r\n\r\n bandit $PARAMS $(inputs.file_list.join(\" \"))\r\n InlineJavascriptRequirement: {}\r\n\r\n inputs:\r\n exit_zero:\r\n label: Exit with zero\r\n doc: Exit with 0, even with results found.\r\n type: boolean\r\n default: true\r\n file_list: string[]\r\n output_file:\r\n label: Output file\r\n doc: Write report to filename.\r\n type: string\r\n default: bandit_report.json\r\n output_format:\r\n label: Output format\r\n doc: Specify output format.\r\n type: string\r\n default: json\r\n source_directory: Directory\r\n verbose: boolean\r\n\r\n outputs:\r\n bandit_report:\r\n type: File\r\n outputBinding:\r\n glob: $(inputs.output_file)\r\n\r\n baseCommand: sh\r\n arguments:\r\n - script.sh\r\n id: bandit_tool", - "version": "0.1" - } - }, - { - "model": "backend.commandlinetool", - "pk": "clone", - "fields": { - "name": "Clone repo", - "definition": "- class: CommandLineTool\r\n\r\n requirements:\r\n DockerRequirement:\r\n dockerPull: alpine/git\r\n InitialWorkDirRequirement:\r\n listing:\r\n - entryname: clone_branch.sh\r\n entry: |-\r\n set -e\r\n\r\n if [ $(inputs.repo_branch) ]; then\r\n echo 'Branch specified: $(inputs.repo_branch). Cloning branch...'\r\n git clone $(inputs.repo_url) -b $(inputs.repo_branch)\r\n else\r\n echo 'No branch specified. Cloning default branch...'\r\n git clone $(inputs.repo_url)\r\n fi\r\n InlineJavascriptRequirement: {}\r\n\r\n inputs:\r\n repo_branch: string\r\n repo_url: string\r\n\r\n outputs:\r\n repo_directory:\r\n type: Directory\r\n outputBinding:\r\n glob: $(inputs.repo_url.split('/').pop().replace('.git',''))\r\n\r\n baseCommand: sh\r\n arguments:\r\n - clone_branch.sh\r\n id: clone_tool", - "version": "0.1" - } - }, - { - "model": "backend.commandlinetool", - "pk": "filter", - "fields": { - "name": "Filter", - "definition": "- class: CommandLineTool\r\n\r\n requirements:\r\n DockerRequirement:\r\n dockerPull: alpine:latest\r\n InitialWorkDirRequirement:\r\n listing:\r\n - entryname: script.sh\r\n entry: |-\r\n cd $(inputs.source_directory.path)\r\n find . -type f -regex \"$(inputs.regex)\" > $HOME/filter.out\r\n InlineJavascriptRequirement: {}\r\n\r\n inputs:\r\n regex: string\r\n source_directory: Directory\r\n\r\n outputs:\r\n file_list:\r\n type: string[]\r\n outputBinding:\r\n glob: filter.out\r\n outputEval: $(self[0].contents.split('\\n').filter(line => line.trim() !== ''))\r\n loadContents: true\r\n\r\n baseCommand: sh\r\n arguments:\r\n - script.sh\r\n id: filter_tool", - "version": "0.1" - } - }, - { - "model": "backend.commandlinetool", - "pk": "flake8", - "fields": { - "name": "Flake8", - "definition": "- class: CommandLineTool\r\n\r\n requirements:\r\n DockerRequirement:\r\n dockerPull: eoepca/appquality-flake8-json:v0.1.0\r\n InitialWorkDirRequirement:\r\n listing:\r\n - entryname: script.sh\r\n entry: |-\r\n cd $(inputs.source_directory.path)\r\n\r\n PARAMS=\"--format=$(inputs.output_format) --output-file=$HOME/$(inputs.output_file)\"\r\n if [ \"$(inputs.exit_zero)\" == \"true\" ] ; then\r\n PARAMS=\"$PARAMS --exit-zero\"\r\n fi\r\n if [ \"$(inputs.verbose)\" == \"true\" ] ; then\r\n PARAMS=\"$PARAMS -v\"\r\n fi\r\n\r\n flake8 $PARAMS $(inputs.file_list.join(\" \"))\r\n InlineJavascriptRequirement: {}\r\n\r\n inputs:\r\n exit_zero:\r\n label: Exit with zero\r\n doc: Force Flake8 to use the exit status code 0 even if there are errors.\r\n type: boolean\r\n default: true\r\n file_list: string[]\r\n output_file:\r\n label: Output file\r\n doc: Redirect all output to the specified file.\r\n type: string\r\n default: flake8_report.json\r\n output_format:\r\n label: Output format\r\n doc: Select the formatter used to display errors to the user.\r\n type: string\r\n default: json\r\n source_directory: Directory\r\n verbose: boolean\r\n\r\n outputs:\r\n flake8_report:\r\n type: File\r\n outputBinding:\r\n glob: $(inputs.output_file)\r\n\r\n baseCommand: sh\r\n arguments:\r\n - script.sh\r\n id: flake8_tool", - "version": "0.1" - } - }, - { - "model": "backend.commandlinetool", - "pk": "ipynb_specs_checker", - "fields": { - "name": "Jupyter Notebook Best Practices Checker", - "definition": "- class: CommandLineTool\r\n\r\n requirements:\r\n DockerRequirement:\r\n dockerPull: python:slim\r\n InitialWorkDirRequirement:\r\n listing:\r\n - entryname: script.sh\r\n entry: |-\r\n cd $(inputs.source_directory.path)\r\n python ~/script.py > ~/ipynb_specs_checker_report.json\r\n exit 0\r\n - entryname: script.py\r\n entry: |-\r\n import json\r\n import os\r\n\r\n schema = '$(inputs.schema)'\r\n\r\n def read_notebook(notebook_path: str):\r\n result = {\"filename\": os.path.relpath(notebook_path, os.path.expanduser('$(inputs.source_directory.path)/..')), \"schema\": schema}\r\n try:\r\n with open(notebook_path) as f:\r\n notebook = json.load(f)\r\n except Exception as e:\r\n result['error'] = (f\"Error reading notebook file: {e}\")\r\n return result\r\n\r\n if \"metadata\" not in notebook:\r\n result['error'] = (\"Notebook does not contain a 'metadata' section.\")\r\n return result\r\n\r\n metadata = notebook[\"metadata\"]\r\n\r\n if schema.lower() == \"eumetsat\":\r\n mandatory_fields = [\"author\", \"title\", \"description\", \"services\"]\r\n optional_fields = [\"image\", \"tags\"]\r\n elif schema.lower() == \"schema.org\":\r\n mandatory_fields = [\"author\", \"name\", \"description\", \"keywords\"]\r\n optional_fields = [\"identifier\", \"image\", \"potentialAction\", \"domain\", \"platform\", \"instruments\", \"tags\", \"license\"]\r\n else:\r\n result['error'] = f\"Unknown schema type: {schema}\"\r\n return result\r\n\r\n errors = []\r\n for field in mandatory_fields:\r\n if field not in metadata:\r\n errors.append(field)\r\n\r\n warnings = []\r\n for field in optional_fields:\r\n if field not in metadata:\r\n warnings.append(field)\r\n\r\n result['valid'] = (len(errors) == 0)\r\n result['missing_mandatory_fields'] = errors\r\n result['missing_optional_fields'] = warnings\r\n\r\n return result\r\n\r\n output = []\r\n\r\n for notebook_path in $(inputs.file_list):\r\n output.append(read_notebook(notebook_path))\r\n\r\n print(json.dumps(output, indent=4))\r\n InlineJavascriptRequirement: {}\r\n\r\n inputs:\r\n schema: string\r\n file_list: string[]\r\n source_directory: Directory\r\n\r\n outputs:\r\n ipynb_specs_checker_report:\r\n type: File\r\n outputBinding:\r\n glob: ipynb_specs_checker_report.json\r\n\r\n baseCommand: sh\r\n arguments:\r\n - script.sh\r\n id: ipynb_specs_checker_tool", - "version": "0.1" - } - }, - { - "model": "backend.commandlinetool", - "pk": "papermill", - "fields": { - "name": "Papermill", - "definition": "- class: CommandLineTool\r\n\r\n requirements:\r\n DockerRequirement:\r\n dockerPull: continuumio/miniconda3\r\n InitialWorkDirRequirement:\r\n listing:\r\n - entryname: script.sh\r\n entry: |-\r\n set -e\r\n\r\n cat ~/err.ipynb > ~/out.ipynb\r\n cd $(inputs.source_directory.path)\r\n\r\n pip install pyyaml\r\n\r\n python ~/script.py $(inputs.notebook_path) | tee ~/environment.yml\r\n\r\n if [ ! -s ~/environment.yml ]; then\r\n echo \"Error: environment.yml is empty. Aborting environment creation.\"\r\n exit 1\r\n fi\r\n\r\n ENV=`grep \"name: \" ~/environment.yml | cut - -d' ' -f2`\r\n\r\n conda env create -f ~/environment.yml\r\n conda run -n $ENV pip install ipykernel papermill\r\n conda run -n $ENV ipython kernel install --user --name $ENV \r\n conda run -n $ENV papermill $(inputs.notebook_path) ~/out.ipynb\r\n - entryname: script.py\r\n entry: |-\r\n import json\r\n import yaml\r\n import sys\r\n\r\n try:\r\n with open('$(inputs.notebook_path)') as f:\r\n nb_json = json.load(f)\r\n except Exception as e:\r\n sys.stderr.write(f\"Failed to load notebook: {e}\\\\n\")\r\n sys.exit(1)\r\n\r\n try:\r\n env = nb_json[\"metadata\"][\"software_requirements\"][\"conda_environment\"]\r\n kernelspec = nb_json[\"metadata\"][\"kernelspec\"][\"name\"]\r\n except KeyError as e:\r\n sys.stderr.write(f\"Missing key in notebook metadata: {e}\\\\n\")\r\n sys.exit(1)\r\n\r\n if env[\"name\"] == kernelspec:\r\n if env.get(\"dependencies\"):\r\n res = yaml.dump({\"name\": env[\"name\"], \"dependencies\": env[\"dependencies\"]})\r\n print(res)\r\n else:\r\n sys.stderr.write(\"No dependencies found in the environment specification.\\\\n\")\r\n sys.exit(1)\r\n else:\r\n sys.stderr.write(\"Kernel name does not match the conda environment name.\\\\n\")\r\n sys.exit(1)\r\n - entryname: err.ipynb\r\n entry: |-\r\n {\r\n \"cells\": [\r\n {\r\n \"cell_type\": \"markdown\",\r\n \"metadata\": {},\r\n \"source\": [\r\n \"Error: The notebook could not be run.\"\r\n ]\r\n }\r\n ],\r\n \"metadata\": {\r\n \"language_info\": {\r\n \"name\": \"python\"\r\n }\r\n },\r\n \"nbformat\": 4,\r\n \"nbformat_minor\": 2\r\n }\r\n\r\n inputs:\r\n notebook_path: string\r\n source_directory: Directory\r\n\r\n outputs:\r\n output_nb:\r\n type: File\r\n outputBinding:\r\n glob: out.ipynb\r\n\r\n baseCommand: sh\r\n arguments:\r\n - script.sh\r\n id: papermill_tool", - "version": "0.1" - } - }, - { - "model": "backend.commandlinetool", - "pk": "pylint", - "fields": { - "name": "Pylint", - "definition": "- class: CommandLineTool\r\n\r\n requirements:\r\n DockerRequirement:\r\n dockerPull: cytopia/pylint\r\n InitialWorkDirRequirement:\r\n listing:\r\n - entryname: script.sh\r\n entry: |-\r\n cd $(inputs.source_directory.path)\r\n\r\n PARAMS=\"--output-format=$(inputs.output_format) --output=$HOME/$(inputs.output_file) --disable=$(inputs.disable)\"\r\n if [ \"$(inputs.exit_zero)\" == \"true\" ] ; then\r\n PARAMS=\"$PARAMS --exit-zero\"\r\n fi\r\n if [ \"$(inputs.errors_only)\" == \"true\" ] ; then\r\n PARAMS=\"$PARAMS -E\"\r\n fi\r\n if [ \"$(inputs.verbose)\" == \"true\" ] ; then\r\n PARAMS=\"$PARAMS -v\"\r\n fi\r\n\r\n pylint $PARAMS $(inputs.file_list.join(\" \"))\r\n InlineJavascriptRequirement: {}\r\n\r\n inputs:\r\n disable: string\r\n errors_only: boolean\r\n exit_zero:\r\n doc: |-\r\n Always return a 0 (non-error) status code, even if lint errors are found. This is primarily useful in continuous integration scripts.\r\n type: boolean\r\n default: true\r\n file_list: string[]\r\n output_file:\r\n doc: Specify an output file.\r\n type: string\r\n default: pylint_report.json\r\n output_format:\r\n doc: |-\r\n Set the output format. Available formats are: text, parseable, colorized, json2 (improved json format), json (old json format) and msvs (visual studio). You can also give a reporter class, e.g. mypackage.mymodule.MyReporterClass.\r\n type: string\r\n default: json\r\n source_directory: Directory\r\n verbose: boolean\r\n\r\n outputs:\r\n pylint_report:\r\n type: File\r\n outputBinding:\r\n glob: $(inputs.output_file)\r\n\r\n baseCommand: sh\r\n arguments:\r\n - script.sh\r\n id: pylint_tool", - "version": "0.1" - } - }, - { - "model": "backend.commandlinetool", - "pk": "ruff", - "fields": { - "name": "Ruff", - "definition": "- class: CommandLineTool\r\n\r\n requirements:\r\n DockerRequirement:\r\n dockerPull: ghcr.io/astral-sh/ruff:alpine\r\n InitialWorkDirRequirement:\r\n listing:\r\n - entryname: script.sh\r\n entry: |-\r\n cd $(inputs.source_directory.path)\r\n\r\n PARAMS=\"--output-format $(inputs.output_format) -o $HOME/$(inputs.output_file)\"\r\n if [ \"$(inputs.exit_zero)\" == \"true\" ] ; then\r\n PARAMS=\"$PARAMS -e\"\r\n fi\r\n if [ \"$(inputs.no_cache)\" == \"true\" ] ; then\r\n PARAMS=\"$PARAMS -n\"\r\n fi\r\n if [ \"$(inputs.verbose)\" == \"true\" ] ; then\r\n PARAMS=\"$PARAMS -v\"\r\n fi\r\n\r\n ruff check $PARAMS $(inputs.file_list.join(\" \"))\r\n InlineJavascriptRequirement: {}\r\n\r\n inputs:\r\n exit_zero:\r\n label: Exit with zero\r\n doc: Exit with status code \"0\", even upon detecting lint violations.\r\n type: boolean\r\n default: true\r\n no_cache:\r\n label: Disable cache\r\n doc: Disable cache reads.\r\n type: boolean\r\n default: true\r\n file_list: string[]\r\n output_file:\r\n label: Output file\r\n doc: Specify file to write the linter output to.\r\n type: string\r\n default: ruff_report.json\r\n output_format:\r\n label: Output format\r\n doc: |-\r\n Output serialization format for violations. Possible values: concise, full, json, json-lines, junit, grouped, github, gitlab, pylint, rdjson, azure, sarif.\r\n type: string\r\n default: json\r\n source_directory: Directory\r\n verbose: boolean\r\n\r\n outputs:\r\n ruff_report:\r\n type: File\r\n outputBinding:\r\n glob: $(inputs.output_file)\r\n\r\n baseCommand: sh\r\n arguments:\r\n - script.sh\r\n id: ruff_tool", - "version": "0.1" - } - }, - { - "model": "backend.commandlinetool", - "pk": "save", - "fields": { - "name": "Save report", - "definition": "- class: CommandLineTool\r\n\r\n requirements:\r\n DockerRequirement:\r\n dockerPull: curlimages/curl\r\n InlineJavascriptRequirement: {}\r\n\r\n inputs:\r\n name: string\r\n pipeline_id: string\r\n report: File\r\n run_id: string\r\n server_url: string\r\n\r\n outputs: []\r\n\r\n baseCommand: curl\r\n arguments:\r\n - prefix: -X\r\n valueFrom: POST\r\n - prefix: -L\r\n valueFrom: |-\r\n $('http://' + inputs.server_url + '/api/pipelines/' + inputs.pipeline_id + '/runs/' + inputs.run_id + '/jobreports/?name=' + inputs.name)\r\n - prefix: -H\r\n valueFrom: Content-Type:application/json\r\n - prefix: -d\r\n valueFrom: $('@' + inputs.report.path)\r\n id: save_tool", - "version": "0.1" - } - }, - { - "model": "backend.commandlinetool", - "pk": "sonarqube_create_project", - "fields": { - "name": "[SonarQube] Create project", - "definition": "- class: CommandLineTool\r\n\r\n requirements:\r\n DockerRequirement:\r\n dockerPull: curlimages/curl\r\n InlineJavascriptRequirement: {}\r\n\r\n inputs:\r\n sonarqube_project_key:\r\n type: string\r\n sonarqube_project_name:\r\n type: string\r\n sonarqube_server:\r\n type: string\r\n default: sonarqube-sonarqube.sonarqube:9000\r\n sonarqube_token:\r\n type: string\r\n\r\n outputs:\r\n sonarqube_project_key:\r\n type: string\r\n outputBinding:\r\n outputEval: $(inputs.sonarqube_project_key)\r\n sonarqube_server:\r\n type: string\r\n outputBinding:\r\n outputEval: $(inputs.sonarqube_server)\r\n sonarqube_token:\r\n type: string\r\n outputBinding:\r\n outputEval: $(inputs.sonarqube_token)\r\n\r\n baseCommand:\r\n - curl\r\n arguments:\r\n - prefix: -X\r\n valueFrom: POST\r\n - prefix: -L\r\n valueFrom: $('http://' + inputs.sonarqube_server + '/api/projects/create')\r\n - prefix: -u\r\n valueFrom: $(inputs.sonarqube_token + ':')\r\n - prefix: -d\r\n valueFrom: $('name=' + inputs.sonarqube_project_name)\r\n - prefix: -d\r\n valueFrom: $('project=' + inputs.sonarqube_project_key)\r\n id: sonarqube_create_project_tool", - "version": "0.1" - } - }, - { - "model": "backend.commandlinetool", - "pk": "sonarqube_get_report", - "fields": { - "name": "[SonarQube] Get report", - "definition": "- class: CommandLineTool\r\n\r\n requirements:\r\n DockerRequirement:\r\n dockerPull: curlimages/curl\r\n InlineJavascriptRequirement: {}\r\n\r\n inputs:\r\n sonarqube_project_key:\r\n type: string\r\n sonarqube_server:\r\n type: string\r\n default: sonarqube-sonarqube.sonarqube:9000\r\n sonarqube_token:\r\n type: string\r\n\r\n outputs:\r\n sonarqube_project_key:\r\n type: string\r\n outputBinding:\r\n outputEval: $(inputs.sonarqube_project_key)\r\n sonarqube_report:\r\n type: File\r\n outputBinding:\r\n glob: sonarqube_report.json\r\n sonarqube_server:\r\n type: string\r\n outputBinding:\r\n outputEval: $(inputs.sonarqube_server)\r\n sonarqube_token:\r\n type: string\r\n outputBinding:\r\n outputEval: $(inputs.sonarqube_token)\r\n stdout: sonarqube_report.json\r\n\r\n baseCommand:\r\n - curl\r\n arguments:\r\n - prefix: -L\r\n valueFrom: |-\r\n $('http://' + inputs.sonarqube_server + '/api/issues/search?components=' + inputs.sonarqube_project_key)\r\n - prefix: -u\r\n valueFrom: $(inputs.sonarqube_token + ':')\r\n id: sonarqube_get_report_tool", - "version": "0.1" - } - }, - { - "model": "backend.commandlinetool", - "pk": "sonarqube_scan", - "fields": { - "name": "[SonarQube] Scan repo", - "definition": "- class: CommandLineTool\r\n\r\n requirements:\r\n DockerRequirement:\r\n dockerPull: sonarsource/sonar-scanner-cli\r\n EnvVarRequirement:\r\n envDef:\r\n SONAR_HOST_URL: $('http://' + inputs.sonarqube_server)\r\n SONAR_TOKEN: $(inputs.sonarqube_token)\r\n InlineJavascriptRequirement: {}\r\n\r\n inputs:\r\n sonarqube_project_key:\r\n type: string\r\n sonarqube_server:\r\n type: string\r\n default: sonarqube-sonarqube.sonarqube:9000\r\n sonarqube_token:\r\n type: string\r\n source_directory:\r\n type: Directory\r\n\r\n outputs:\r\n sonarqube_project_key:\r\n type: string\r\n outputBinding:\r\n glob:\r\n outputEval: $(inputs.sonarqube_project_key)\r\n sonarqube_server:\r\n type: string\r\n outputBinding:\r\n glob:\r\n outputEval: $(inputs.sonarqube_server)\r\n sonarqube_token:\r\n type: string\r\n outputBinding:\r\n glob:\r\n outputEval: $(inputs.sonarqube_token)\r\n\r\n baseCommand:\r\n - sonar-scanner\r\n arguments:\r\n - prefix: -D\r\n valueFrom: $('sonar.projectKey=' + inputs.sonarqube_project_key)\r\n separate: false\r\n # - prefix: -D\r\n # valueFrom: $('sonar.userHome=$HOME')\r\n # separate: false\r\n - prefix: -D\r\n valueFrom: $('sonar.projectBaseDir=' + inputs.source_directory.path + '/../')\r\n separate: false\r\n # - prefix: -D\r\n # valueFrom: $('sonar.source=$HOME' /*+ inputs.source_directory.path*/)\r\n # separate: false\r\n - prefix: -X\r\n id: sonarqube_scan_tool", - "version": "0.1" - } - }, - { - "model": "backend.commandlinetool", - "pk": "trivy", - "fields": { - "name": "Trivy", - "definition": "- class: CommandLineTool\r\n\r\n requirements:\r\n DockerRequirement:\r\n dockerPull: aquasec/trivy:latest\r\n\r\n inputs:\r\n image: string\r\n\r\n outputs:\r\n trivy_report:\r\n type: File\r\n outputBinding:\r\n glob: trivy_report.json\r\n\r\n baseCommand: trivy\r\n arguments:\r\n - image\r\n - prefix: -f\r\n valueFrom: json\r\n - prefix: -o\r\n valueFrom: trivy_report.json\r\n - $(inputs.image)\r\n id: trivy_tool", - "version": "0.1" - } - } -] \ No newline at end of file diff --git a/backend/backend/fixtures/2025-05-12.json b/backend/backend/fixtures/2025-05-12.json deleted file mode 100644 index b542684..0000000 --- a/backend/backend/fixtures/2025-05-12.json +++ /dev/null @@ -1,642 +0,0 @@ -[ - { - "model": "backend.pipeline", - "pk": 6, - "fields": { - "name": "Python pipeline", - "description": "Runs a series of static analysis tools on python files", - "template": "#!/usr/bin/env cwltool\r\n\r\n$graph:\r\n- class: Workflow\r\n\r\n requirements:\r\n SubworkflowFeatureRequirement: {}\r\n\r\n inputs:\r\n{%- for tool in subworkflows %}\r\n {%- for tool_name, params in tool.user_params.items() %}\r\n {%- for param_name, param in params.items() %}\r\n {{ tool.slug }}.{{ tool_name }}.{{ param_name }}:\r\n label: \"{{ param.label or param_name }}\"\r\n {%- if param.doc %}\r\n doc: |-\r\n {{ param.doc }}\r\n {%- endif %}\r\n type: {{ param.type }}\r\n default: {{ param.default | tojson }}\r\n {%- endfor %}\r\n {%- endfor %}\r\n{%- endfor %}\r\n branch:\r\n type: string\r\n default: ''\r\n pipeline_id: string\r\n run_id: string\r\n server_url: string\r\n\r\n outputs: []\r\n\r\n steps:\r\n {% for tool in subworkflows %}{{ tool.pipeline_step | indent(4) }}\r\n {% endfor %}\r\n id: main\r\n{% for tool in subworkflows %}\r\n{{ tool.definition }}\r\n{% endfor %}\r\ncwlVersion: v1.0", - "default_inputs": {}, - "owner": null, - "created_at": null, - "edited_at": null, - "version": "0.1", - "tools": [ - "clone_subworkflow", - "bandit_subworkflow", - "flake8_subworkflow", - "pylint_subworkflow", - "ruff_subworkflow" - ] - } - }, - { - "model": "backend.pipeline", - "pk": 8, - "fields": { - "name": "Notebook pipeline", - "description": "Runs a static analysis and then executes Jupyter Notebooks files (ipynb).", - "template": "#!/usr/bin/env cwltool\r\n\r\n$graph:\r\n- class: Workflow\r\n\r\n requirements:\r\n SubworkflowFeatureRequirement: {}\r\n\r\n inputs:\r\n{%- for tool in subworkflows %}\r\n {%- for tool_name, params in tool.user_params.items() %}\r\n {%- for param_name, param in params.items() %}\r\n {{ tool.slug }}.{{ tool_name }}.{{ param_name }}:\r\n label: \"{{ param.label or param_name }}\"\r\n {%- if param.doc %}\r\n doc: |-\r\n {{ param.doc }}\r\n {%- endif %}\r\n type: {{ param.type }}\r\n default: {{ param.default | tojson }}\r\n {%- endfor %}\r\n {%- endfor %}\r\n{%- endfor %}\r\n branch:\r\n type: string\r\n default: ''\r\n pipeline_id: string\r\n run_id: string\r\n server_url: string\r\n\r\n outputs: []\r\n\r\n steps:\r\n {% for tool in subworkflows %}{{ tool.pipeline_step | indent(4) }}\r\n {% endfor %}\r\n id: main\r\n{% for tool in subworkflows %}\r\n{{ tool.definition }}\r\n{% endfor %}\r\ncwlVersion: v1.0", - "default_inputs": {}, - "owner": null, - "created_at": null, - "edited_at": null, - "version": "0.1", - "tools": [ - "clone_subworkflow", - "notebook-bp-validator_subworkflow", - "papermill_subworkflow", - "ruff_ipynb_subworkflow" - ] - } - }, - { - "model": "backend.pipeline", - "pk": 11, - "fields": { - "name": "Docker pipeline", - "description": "Finds vulnerabilities and misconfigurations in Docker images.", - "template": "#!/usr/bin/env cwltool\r\n\r\n$graph:\r\n- class: Workflow\r\n\r\n requirements:\r\n SubworkflowFeatureRequirement: {}\r\n\r\n inputs:\r\n{%- for tool in subworkflows %}\r\n {%- for tool_name, params in tool.user_params.items() %}\r\n {%- for param_name, param in params.items() %}\r\n {{ tool.slug }}.{{ tool_name }}.{{ param_name }}:\r\n label: \"{{ param.label or param_name }}\"\r\n {%- if param.doc %}\r\n doc: |-\r\n {{ param.doc }}\r\n {%- endif %}\r\n type: {{ param.type }}\r\n default: {{ param.default | tojson }}\r\n {%- endfor %}\r\n {%- endfor %}\r\n{%- endfor %}\r\n pipeline_id: string\r\n run_id: string\r\n server_url: string\r\n\r\n outputs: []\r\n\r\n steps:\r\n {% for tool in subworkflows %}{{ tool.pipeline_step | indent(4) }}\r\n {% endfor %}\r\n id: main\r\n{% for tool in subworkflows %}\r\n{{ tool.definition }}\r\n{% endfor %}\r\ncwlVersion: v1.0", - "default_inputs": {}, - "owner": null, - "created_at": null, - "edited_at": null, - "version": "0.1", - "tools": [ - "trivy_subworkflow" - ] - } - }, - { - "model": "backend.tag", - "pk": 1, - "fields": { - "name": "asset: python" - } - }, - { - "model": "backend.tag", - "pk": 2, - "fields": { - "name": "asset: other" - } - }, - { - "model": "backend.tag", - "pk": 3, - "fields": { - "name": "asset: cwl" - } - }, - { - "model": "backend.tag", - "pk": 4, - "fields": { - "name": "asset: notebook" - } - }, - { - "model": "backend.tag", - "pk": 5, - "fields": { - "name": "type: best practice" - } - }, - { - "model": "backend.tag", - "pk": 6, - "fields": { - "name": "type: app quality" - } - }, - { - "model": "backend.tag", - "pk": 7, - "fields": { - "name": "type: app performance" - } - }, - { - "model": "backend.tag", - "pk": 8, - "fields": { - "name": "type: init" - } - }, - { - "model": "backend.tag", - "pk": 9, - "fields": { - "name": "asset: docker" - } - }, - { - "model": "backend.subworkflow", - "pk": "ap_validator_subworkflow", - "fields": { - "name": "Application Package Validator", - "description": "Validation tool for checking OGC compliance of CWL files for application packages.", - "pipeline_step": "ap_validator_subworkflow:\r\n in:\r\n ap_validator.detail: ap_validator_subworkflow.ap_validator.detail\r\n ap_validator.entry_point: ap_validator_subworkflow.ap_validator.entry_point\r\n filter.regex: ap_validator_subworkflow.filter.regex\r\n pipeline_id: pipeline_id\r\n run_id: run_id\r\n server_url: server_url\r\n source_directory: clone_step/repo_directory\r\n run: '#ap_validator_subworkflow'\r\n out: []", - "definition": "- class: Workflow\r\n\r\n requirements:\r\n ScatterFeatureRequirement: {}\r\n\r\n inputs:\r\n name:\r\n type: string\r\n default: ap_validator\r\n ap_validator.detail: string\r\n ap_validator.entry_point: string\r\n filter.regex: string\r\n pipeline_id: string\r\n run_id: string\r\n server_url: string\r\n source_directory: Directory\r\n\r\n outputs:\r\n []\r\n\r\n steps:\r\n filter_ap_validator_step:\r\n in:\r\n regex: filter.regex\r\n source_directory: source_directory\r\n run: '#filter_tool'\r\n out:\r\n - file_list\r\n ap_validator_step:\r\n in:\r\n file_path: filter_ap_validator_step/file_list\r\n source_directory: source_directory\r\n detail: ap_validator.detail\r\n entry_point: ap_validator.entry_point\r\n scatter: file_path\r\n run: '#ap_validator_tool'\r\n out:\r\n - ap_validator_report\r\n save_ap_validator_step:\r\n in:\r\n name: name\r\n pipeline_id: pipeline_id\r\n report: ap_validator_step/ap_validator_report\r\n run_id: run_id\r\n server_url: server_url\r\n scatter: report\r\n run: '#save_tool'\r\n out: []\r\n id: ap_validator_subworkflow\r\n{% for tool in tools %}{{ tool.definition }}\r\n{% endfor %}", - "user_params": { - "filter": { - "regex": { - "type": "string", - "label": "regex", - "default": ".*\\.cwl" - } - }, - "ap_validator": { - "detail": { - "doc": "Output detail (none|errors|hints|all). Default: hints", - "type": "string", - "label": "Detail", - "default": "hints" - }, - "entry_point": { - "doc": "Name of entry point (Workflow or CommandLineTool)", - "type": "string", - "label": "Entry point", - "default": "main" - } - } - }, - "version": "0.1", - "tags": [ - 3, - 5 - ], - "tools": [ - "ap_validator", - "filter", - "save" - ] - } - }, - { - "model": "backend.subworkflow", - "pk": "bandit_subworkflow", - "fields": { - "name": "Bandit", - "description": "Bandit - Bandit is a tool designed to find common security issues in Python code", - "pipeline_step": "bandit_subworkflow:\r\n in:\r\n bandit.verbose: bandit_subworkflow.bandit.verbose\r\n filter.regex: bandit_subworkflow.filter.regex\r\n pipeline_id: pipeline_id\r\n run_id: run_id\r\n server_url: server_url\r\n source_directory: clone_step/repo_directory\r\n run: '#bandit_subworkflow'\r\n out: []", - "definition": "- class: Workflow\r\n\r\n inputs:\r\n name:\r\n type: string\r\n default: bandit\r\n bandit.verbose: boolean\r\n filter.regex: string\r\n pipeline_id: string\r\n run_id: string\r\n server_url: string\r\n source_directory: Directory\r\n\r\n outputs:\r\n bandit_report:\r\n type: File\r\n outputSource: bandit_step/bandit_report\r\n\r\n steps:\r\n bandit_step:\r\n in:\r\n file_list: filter_bandit_step/file_list\r\n source_directory: source_directory\r\n verbose: bandit.verbose\r\n run: '#bandit_tool'\r\n out:\r\n - bandit_report\r\n filter_bandit_step:\r\n in:\r\n regex: filter.regex\r\n source_directory: source_directory\r\n run: '#filter_tool'\r\n out:\r\n - file_list\r\n save_bandit_step:\r\n in:\r\n name: name\r\n pipeline_id: pipeline_id\r\n report: bandit_step/bandit_report\r\n run_id: run_id\r\n server_url: server_url\r\n run: '#save_tool'\r\n out: []\r\n id: bandit_subworkflow\r\n{% for tool in tools %}{{ tool.definition }}\r\n{% endfor %}", - "user_params": { - "bandit": { - "verbose": { - "doc": "Output extra information like excluded and included files.", - "type": "boolean", - "label": "Verbose", - "default": false - } - }, - "filter": { - "regex": { - "type": "string", - "label": "regex", - "default": ".*\\.py" - } - } - }, - "version": "0.1", - "tags": [ - 1, - 6 - ], - "tools": [ - "bandit", - "filter", - "save" - ] - } - }, - { - "model": "backend.subworkflow", - "pk": "clone_subworkflow", - "fields": { - "name": "Clone repo", - "description": "git-clone - Clone a repository into a new directory", - "pipeline_step": "clone_step:\r\n in:\r\n clone.repo_branch: clone_subworkflow.clone.repo_branch\r\n clone.repo_url: clone_subworkflow.clone.repo_url\r\n run: '#clone_subworkflow'\r\n out:\r\n - repo_directory", - "definition": "- class: Workflow\r\n\r\n inputs:\r\n clone.repo_branch:\r\n type: string\r\n default: ''\r\n clone.repo_url: string\r\n\r\n outputs:\r\n repo_directory:\r\n type: Directory\r\n outputSource: clone_tool_step/repo_directory\r\n\r\n steps:\r\n clone_tool_step:\r\n in:\r\n repo_branch: clone.repo_branch\r\n repo_url: clone.repo_url\r\n run: '#clone_tool'\r\n out:\r\n - repo_directory\r\n id: clone_subworkflow\r\n{% for tool in tools %}{{ tool.definition }}\r\n{% endfor %}", - "user_params": { - "clone": { - "repo_url": { - "doc": "URL to the repository to clone.", - "type": "string", - "label": "Repo URL", - "default": "" - }, - "repo_branch": { - "doc": "Branch to checkout instead of the remote HEAD.", - "type": "string", - "label": "Repo branch", - "default": "" - } - } - }, - "version": "0.1", - "tags": [ - 2, - 8 - ], - "tools": [ - "clone" - ] - } - }, - { - "model": "backend.subworkflow", - "pk": "flake8_subworkflow", - "fields": { - "name": "Flake8", - "description": "flake8 - Style guide enforcement tool for Python", - "pipeline_step": "flake8_subworkflow:\r\n in:\r\n filter.regex: flake8_subworkflow.filter.regex\r\n flake8.verbose: flake8_subworkflow.flake8.verbose\r\n pipeline_id: pipeline_id\r\n run_id: run_id\r\n server_url: server_url\r\n source_directory: clone_step/repo_directory\r\n run: '#flake8_subworkflow'\r\n out: []", - "definition": "- class: Workflow\r\n\r\n inputs:\r\n name:\r\n type: string\r\n default: flake8\r\n filter.regex: string\r\n flake8.verbose: boolean\r\n pipeline_id: string\r\n run_id: string\r\n server_url: string\r\n source_directory: Directory\r\n\r\n outputs:\r\n flake8_report:\r\n type: File\r\n outputSource: flake8_step/flake8_report\r\n\r\n steps:\r\n filter_flake8_step:\r\n in:\r\n regex: filter.regex\r\n source_directory: source_directory\r\n run: '#filter_tool'\r\n out:\r\n - file_list\r\n flake8_step:\r\n in:\r\n file_list: filter_flake8_step/file_list\r\n source_directory: source_directory\r\n verbose: flake8.verbose\r\n run: '#flake8_tool'\r\n out:\r\n - flake8_report\r\n save_flake8_step:\r\n in:\r\n name: name\r\n pipeline_id: pipeline_id\r\n report: flake8_step/flake8_report\r\n run_id: run_id\r\n server_url: server_url\r\n run: '#save_tool'\r\n out: []\r\n id: flake8_subworkflow\r\n{% for tool in tools %}{{ tool.definition }}\r\n{% endfor %}", - "user_params": { - "filter": { - "regex": { - "type": "string", - "label": "regex", - "default": ".*\\.py" - } - }, - "flake8": { - "verbose": { - "doc": "Increase the verbosity of Flake8’s output.", - "type": "boolean", - "label": "Verbose", - "default": false - } - } - }, - "version": "0.1", - "tags": [ - 1, - 5 - ], - "tools": [ - "filter", - "flake8", - "save" - ] - } - }, - { - "model": "backend.subworkflow", - "pk": "notebook-bp-validator_subworkflow", - "fields": { - "name": "Jupyter Notebook Best Practices Validator", - "description": "This tool aims at validating the notebooks against the CEOS Jupyter Notebook Best Practice v1.1 document.", - "pipeline_step": "notebook-bp-validator_subworkflow:\r\n in:\r\n filter.regex: notebook-bp-validator_subworkflow.filter.regex\r\n pipeline_id: pipeline_id\r\n notebook-bp-validator.abspath: notebook-bp-validator_subworkflow.notebook-bp-validator.abspath\r\n notebook-bp-validator.schema: notebook-bp-validator_subworkflow.notebook-bp-validator.schema\r\n run_id: run_id\r\n server_url: server_url\r\n source_directory: clone_step/repo_directory\r\n run: '#notebook-bp-validator_subworkflow'\r\n out: []", - "definition": "- class: Workflow\r\n\r\n inputs:\r\n name:\r\n type: string\r\n default: notebook-bp-validator\r\n filter.regex: string\r\n pipeline_id: string\r\n notebook-bp-validator.abspath: boolean\r\n notebook-bp-validator.schema: string\r\n run_id: string\r\n server_url: string\r\n source_directory: Directory\r\n\r\n outputs:\r\n notebook-bp-validator_report:\r\n type: File\r\n outputSource: notebook-bp-validator_step/notebook-bp-validator_report\r\n\r\n steps:\r\n filter_notebook-bp-validator_step:\r\n in:\r\n regex: filter.regex\r\n source_directory: source_directory\r\n run: '#filter_tool'\r\n out:\r\n - file_list\r\n notebook-bp-validator_step:\r\n in:\r\n abspath: notebook-bp-validator.abspath \r\n file_list: filter_notebook-bp-validator_step/file_list\r\n source_directory: source_directory\r\n schema: notebook-bp-validator.schema\r\n run: '#notebook-bp-validator_tool'\r\n out:\r\n - notebook-bp-validator_report\r\n save_notebook-bp-validator_step:\r\n in:\r\n name: name\r\n pipeline_id: pipeline_id\r\n report: notebook-bp-validator_step/notebook-bp-validator_report\r\n run_id: run_id\r\n server_url: server_url\r\n run: '#save_tool'\r\n out: []\r\n id: notebook-bp-validator_subworkflow\r\n{% for tool in tools %}{{ tool.definition }}\r\n{% endfor %}", - "user_params": { - "filter": { - "regex": { - "type": "string", - "label": "regex", - "default": ".*\\.ipynb" - } - }, - "notebook-bp-validator": { - "schema": { - "doc": "Supported values: 'eumetsat' or 'schema.org'", - "type": "string", - "label": "Schema", - "default": "eumetsat" - }, - "abspath": { - "doc": "Uses absolute paths in output.", - "type": "boolean", - "label": "Absolute path", - "default": false - } - } - }, - "version": "0.1", - "tags": [ - 4, - 5 - ], - "tools": [ - "filter", - "notebook-bp-validator", - "save" - ] - } - }, - { - "model": "backend.subworkflow", - "pk": "papermill_subworkflow", - "fields": { - "name": "Papermill", - "description": "Papermill is a tool for parameterizing and executing Jupyter Notebooks.", - "pipeline_step": "papermill_subworkflow:\r\n in:\r\n filter.regex: papermill_subworkflow.filter.regex\r\n pipeline_id: pipeline_id\r\n run_id: run_id\r\n server_url: server_url\r\n source_directory: clone_step/repo_directory\r\n run: '#papermill_subworkflow'\r\n out: []", - "definition": "- class: Workflow\r\n\r\n requirements:\r\n ScatterFeatureRequirement: {}\r\n\r\n inputs:\r\n name:\r\n type: string\r\n default: papermill\r\n filter.regex: string\r\n pipeline_id: string\r\n run_id: string\r\n server_url: string\r\n source_directory: Directory\r\n\r\n outputs:\r\n []\r\n\r\n steps:\r\n filter_papermill_step:\r\n in:\r\n regex: filter.regex\r\n source_directory: source_directory\r\n run: '#filter_tool'\r\n out:\r\n - file_list\r\n papermill_step:\r\n in:\r\n notebook_path: filter_papermill_step/file_list\r\n source_directory: source_directory\r\n scatter: notebook_path\r\n run: '#papermill_tool'\r\n out:\r\n - output_nb\r\n save_papermill_step:\r\n in:\r\n name: name\r\n pipeline_id: pipeline_id\r\n report: papermill_step/output_nb\r\n run_id: run_id\r\n server_url: server_url\r\n scatter: report\r\n run: '#save_tool'\r\n out: []\r\n id: papermill_subworkflow\r\n{% for tool in tools %}{{ tool.definition }}\r\n{% endfor %}", - "user_params": { - "filter": { - "regex": { - "type": "string", - "label": "regex", - "default": ".*\\.ipynb" - } - } - }, - "version": "0.1", - "tags": [ - 4, - 7 - ], - "tools": [ - "filter", - "papermill", - "save" - ] - } - }, - { - "model": "backend.subworkflow", - "pk": "pylint_subworkflow", - "fields": { - "name": "Pylint", - "description": "pylint - Static code analyser tool for Python", - "pipeline_step": "pylint_subworkflow:\r\n in:\r\n filter.regex: pylint_subworkflow.filter.regex\r\n pipeline_id: pipeline_id\r\n pylint.disable: pylint_subworkflow.pylint.disable\r\n pylint.errors_only: pylint_subworkflow.pylint.errors_only\r\n pylint.verbose: pylint_subworkflow.pylint.verbose\r\n run_id: run_id\r\n server_url: server_url\r\n source_directory: clone_step/repo_directory\r\n run: '#pylint_subworkflow'\r\n out: []", - "definition": "- class: Workflow\r\n\r\n inputs:\r\n name:\r\n type: string\r\n default: pylint\r\n filter.regex: string\r\n pipeline_id: string\r\n pylint.disable: string\r\n pylint.errors_only: boolean\r\n pylint.verbose: boolean\r\n run_id: string\r\n server_url: string\r\n source_directory: Directory\r\n\r\n outputs:\r\n pylint_report:\r\n type: File\r\n outputSource: pylint_step/pylint_report\r\n\r\n steps:\r\n filter_pylint_step:\r\n in:\r\n regex: filter.regex\r\n source_directory: source_directory\r\n run: '#filter_tool'\r\n out:\r\n - file_list\r\n pylint_step:\r\n in:\r\n disable: pylint.disable\r\n errors_only: pylint.errors_only\r\n file_list: filter_pylint_step/file_list\r\n source_directory: source_directory\r\n verbose: pylint.verbose\r\n run: '#pylint_tool'\r\n out:\r\n - pylint_report\r\n save_pylint_step:\r\n in:\r\n name: name\r\n pipeline_id: pipeline_id\r\n report: pylint_step/pylint_report\r\n run_id: run_id\r\n server_url: server_url\r\n run: '#save_tool'\r\n out: []\r\n id: pylint_subworkflow\r\n{% for tool in tools %}{{ tool.definition }}\r\n{% endfor %}", - "user_params": { - "filter": { - "regex": { - "type": "string", - "label": "regex", - "default": ".*\\.py" - } - }, - "pylint": { - "disable": { - "doc": "Disable the message, report, category or checker with the given id(s).", - "type": "string", - "label": "Disable IDs", - "default": "E0401" - }, - "verbose": { - "doc": "In verbose mode, extra non-checker-related info will be displayed.", - "type": "boolean", - "label": "Verbose", - "default": false - }, - "errors_only": { - "doc": "In error mode, messages with a category besides ERROR or FATAL are suppressed, and no reports are done by default. Error mode is compatible with disabling specific errors.", - "type": "boolean", - "label": "Errors only", - "default": false - } - } - }, - "version": "0.1", - "tags": [ - 1, - 5 - ], - "tools": [ - "filter", - "pylint", - "save" - ] - } - }, - { - "model": "backend.subworkflow", - "pk": "ruff_ipynb_subworkflow", - "fields": { - "name": "Ruff - Notebook", - "description": "Ruff - An extremely fast Python linter and code formatter, written in Rust", - "pipeline_step": "ruff_ipynb_subworkflow:\r\n in:\r\n filter.regex: ruff_ipynb_subworkflow.filter.regex\r\n pipeline_id: pipeline_id\r\n ruff.verbose: ruff_ipynb_subworkflow.ruff.verbose\r\n run_id: run_id\r\n server_url: server_url\r\n source_directory: clone_step/repo_directory\r\n run: '#ruff_ipynb_subworkflow'\r\n out: []", - "definition": "- class: Workflow\r\n\r\n inputs:\r\n name:\r\n type: string\r\n default: ruff\r\n filter.regex: string\r\n pipeline_id: string\r\n ruff.verbose: boolean\r\n run_id: string\r\n server_url: string\r\n source_directory: Directory\r\n\r\n outputs:\r\n ruff_report:\r\n type: File\r\n outputSource: ruff_step/ruff_report\r\n\r\n steps:\r\n filter_ruff_step:\r\n in:\r\n regex: filter.regex\r\n source_directory: source_directory\r\n run: '#filter_tool'\r\n out:\r\n - file_list\r\n ruff_step:\r\n in:\r\n file_list: filter_ruff_step/file_list\r\n source_directory: source_directory\r\n verbose: ruff.verbose\r\n run: '#ruff_tool'\r\n out:\r\n - ruff_report\r\n save_ruff_step:\r\n in:\r\n name: name\r\n pipeline_id: pipeline_id\r\n report: ruff_step/ruff_report\r\n run_id: run_id\r\n server_url: server_url\r\n run: '#save_tool'\r\n out: []\r\n id: ruff_ipynb_subworkflow\r\n{% for tool in tools %}{{ tool.definition }}\r\n{% endfor %}", - "user_params": { - "ruff": { - "verbose": { - "doc": "Enable verbose logging.", - "type": "boolean", - "label": "Verbose", - "default": false - } - }, - "filter": { - "regex": { - "type": "string", - "label": "regex", - "default": ".*\\.ipynb" - } - } - }, - "version": "0.1", - "tags": [ - 4, - 5 - ], - "tools": [ - "filter", - "ruff", - "save" - ] - } - }, - { - "model": "backend.subworkflow", - "pk": "ruff_subworkflow", - "fields": { - "name": "Ruff", - "description": "Ruff - An extremely fast Python linter and code formatter, written in Rust", - "pipeline_step": "ruff_subworkflow:\r\n in:\r\n filter.regex: ruff_subworkflow.filter.regex\r\n pipeline_id: pipeline_id\r\n ruff.verbose: ruff_subworkflow.ruff.verbose\r\n run_id: run_id\r\n server_url: server_url\r\n source_directory: clone_step/repo_directory\r\n run: '#ruff_subworkflow'\r\n out: []", - "definition": "- class: Workflow\r\n\r\n inputs:\r\n name:\r\n type: string\r\n default: ruff\r\n filter.regex: string\r\n pipeline_id: string\r\n ruff.verbose: boolean\r\n run_id: string\r\n server_url: string\r\n source_directory: Directory\r\n\r\n outputs:\r\n ruff_report:\r\n type: File\r\n outputSource: ruff_step/ruff_report\r\n\r\n steps:\r\n filter_ruff_step:\r\n in:\r\n regex: filter.regex\r\n source_directory: source_directory\r\n run: '#filter_tool'\r\n out:\r\n - file_list\r\n ruff_step:\r\n in:\r\n file_list: filter_ruff_step/file_list\r\n source_directory: source_directory\r\n verbose: ruff.verbose\r\n run: '#ruff_tool'\r\n out:\r\n - ruff_report\r\n save_ruff_step:\r\n in:\r\n name: name\r\n pipeline_id: pipeline_id\r\n report: ruff_step/ruff_report\r\n run_id: run_id\r\n server_url: server_url\r\n run: '#save_tool'\r\n out: []\r\n id: ruff_subworkflow\r\n{% for tool in tools %}{{ tool.definition }}\r\n{% endfor %}", - "user_params": { - "ruff": { - "verbose": { - "doc": "Enable verbose logging.", - "type": "boolean", - "label": "Verbose", - "default": false - } - }, - "filter": { - "regex": { - "type": "string", - "label": "regex", - "default": ".*\\.py" - } - } - }, - "version": "0.1", - "tags": [ - 1, - 5 - ], - "tools": [ - "filter", - "ruff", - "save" - ] - } - }, - { - "model": "backend.subworkflow", - "pk": "sonarqube", - "fields": { - "name": "SonarQube", - "description": "SonarQube - Code Quality, Security & Static Analysis Tool\r\n\r\nThis tool creates a project in our internal SonarQube server, sends it the code for analysis, and then retrieves the analysis results for storage in the database.", - "pipeline_step": "sonarqube_workflow_step:\r\n in:\r\n pipeline_id: pipeline_id\r\n repo_path: clone_step/repo_directory\r\n run_id: run_id\r\n server_url: server_url\r\n sonarqube_project_key: sonarqube_project_key\r\n sonarqube_project_name: sonarqube_project_name\r\n sonarqube_server: sonarqube_server\r\n sonarqube_token: sonarqube_token\r\n run: '#sonarqube_workflow'\r\n out: []", - "definition": "- class: Workflow\r\n\r\n inputs:\r\n name:\r\n type: string\r\n default: sonarqube\r\n pipeline_id:\r\n type: string\r\n repo_path:\r\n type: Directory\r\n run_id:\r\n type: string\r\n server_url:\r\n type: string\r\n sonarqube_project_key:\r\n type: string\r\n sonarqube_project_name:\r\n type: string\r\n sonarqube_server:\r\n type: string\r\n default: sonarqube-sonarqube.sonarqube:9000\r\n sonarqube_token:\r\n type: string\r\n\r\n outputs: []\r\n\r\n steps:\r\n save_sonarqube_step:\r\n in:\r\n name: name\r\n pipeline_id: pipeline_id\r\n report: sonarqube_get_report_step/sonarqube_report\r\n run_id: run_id\r\n server_url: server_url\r\n run: '#save_tool'\r\n out: []\r\n sonarqube_create_project_step:\r\n in:\r\n sonarqube_project_key: sonarqube_project_key\r\n sonarqube_project_name: sonarqube_project_name\r\n sonarqube_server: sonarqube_server\r\n sonarqube_token: sonarqube_token\r\n run: '#sonarqube_create_project_tool'\r\n out:\r\n - sonarqube_project_key\r\n - sonarqube_server\r\n - sonarqube_token\r\n sonarqube_get_report_step:\r\n in:\r\n sonarqube_project_key: sonarqube_scan_step/sonarqube_project_key\r\n sonarqube_server: sonarqube_scan_step/sonarqube_server\r\n sonarqube_token: sonarqube_scan_step/sonarqube_token\r\n run: '#sonarqube_get_report_tool'\r\n out:\r\n - sonarqube_project_key\r\n - sonarqube_server\r\n - sonarqube_token\r\n - sonarqube_report\r\n sonarqube_scan_step:\r\n in:\r\n sonarqube_project_key: sonarqube_create_project_step/sonarqube_project_key\r\n sonarqube_server: sonarqube_create_project_step/sonarqube_server\r\n sonarqube_token: sonarqube_create_project_step/sonarqube_token\r\n source_directory: repo_path\r\n run: '#sonarqube_scan_tool'\r\n out:\r\n - sonarqube_project_key\r\n - sonarqube_server\r\n - sonarqube_token\r\n id: sonarqube_workflow\r\n{% for tool in tools %}{{ tool.definition }}\r\n{% endfor %}", - "user_params": {}, - "version": "0.1", - "tags": [], - "tools": [ - "save", - "sonarqube_create_project", - "sonarqube_get_report", - "sonarqube_scan" - ] - } - }, - { - "model": "backend.subworkflow", - "pk": "trivy_subworkflow", - "fields": { - "name": "Trivy", - "description": "The all-in-one open source security scanner\r\nUse Trivy to find vulnerabilities (CVE) & misconfigurations (IaC) across code repositories, binary artifacts, container images, Kubernetes clusters, and more.", - "pipeline_step": "trivy_subworkflow:\r\n in:\r\n pipeline_id: pipeline_id\r\n run_id: run_id\r\n server_url: server_url\r\n trivy.image: trivy_subworkflow.trivy.image\r\n run: '#trivy_subworkflow'\r\n out: []", - "definition": "- class: Workflow\r\n\r\n inputs:\r\n name:\r\n type: string\r\n default: trivy\r\n pipeline_id: string\r\n run_id: string\r\n server_url: string\r\n trivy.image: string\r\n\r\n outputs: []\r\n\r\n steps:\r\n save_step:\r\n in:\r\n name: name\r\n pipeline_id: pipeline_id\r\n report: trivy_step/trivy_report\r\n run_id: run_id\r\n server_url: server_url\r\n run: '#save_tool'\r\n out: []\r\n trivy_step:\r\n in:\r\n image: trivy.image\r\n run: '#trivy_tool'\r\n out:\r\n - trivy_report\r\n id: trivy_subworkflow\r\n{% for tool in tools %}{{ tool.definition }}\r\n{% endfor %}", - "user_params": { - "trivy": { - "image": { - "type": "string", - "label": "Docker image", - "default": "alpine/git" - } - } - }, - "version": "0.1", - "tags": [ - 6, - 9 - ], - "tools": [ - "save", - "trivy" - ] - } - }, - { - "model": "backend.commandlinetool", - "pk": "ap_validator", - "fields": { - "name": "Application Package Validator", - "definition": "- class: CommandLineTool\r\n\r\n requirements:\r\n DockerRequirement:\r\n dockerPull: nexus.spaceapplications.com/repository/docker-eoepca/ap_validator:2025-03-05.1\r\n InitialWorkDirRequirement:\r\n listing:\r\n - entryname: script.sh\r\n entry: |-\r\n cd $(inputs.source_directory.path)\r\n\r\n ap-validator \\\\\r\n --format json \\\\\r\n --detail '$(inputs.detail)' \\\\\r\n --entry-point '$(inputs.entry_point)' \\\\\r\n '$(inputs.file_path)' > ~/ap_validator_report.json\r\n\r\n exit 0\r\n InlineJavascriptRequirement: {}\r\n\r\n inputs:\r\n file_path: string\r\n source_directory: Directory\r\n detail: string\r\n entry_point: string\r\n\r\n outputs:\r\n ap_validator_report:\r\n type: File\r\n outputBinding:\r\n glob: ap_validator_report.json\r\n\r\n baseCommand: sh\r\n arguments:\r\n - script.sh\r\n id: ap_validator_tool", - "version": "0.1" - } - }, - { - "model": "backend.commandlinetool", - "pk": "bandit", - "fields": { - "name": "Bandit", - "definition": "- class: CommandLineTool\r\n\r\n requirements:\r\n DockerRequirement:\r\n dockerPull: cytopia/bandit\r\n InitialWorkDirRequirement:\r\n listing:\r\n - entryname: script.sh\r\n entry: |-\r\n cd $(inputs.source_directory.path)\r\n\r\n PARAMS=\"-f $(inputs.output_format) -o $HOME/$(inputs.output_file)\"\r\n if [ \"$(inputs.exit_zero)\" == \"true\" ] ; then\r\n PARAMS=\"$PARAMS --exit-zero\"\r\n fi\r\n if [ \"$(inputs.verbose)\" == \"true\" ] ; then\r\n PARAMS=\"$PARAMS -v\"\r\n fi\r\n\r\n bandit $PARAMS $(inputs.file_list.join(\" \"))\r\n InlineJavascriptRequirement: {}\r\n\r\n inputs:\r\n exit_zero:\r\n label: Exit with zero\r\n doc: Exit with 0, even with results found.\r\n type: boolean\r\n default: true\r\n file_list: string[]\r\n output_file:\r\n label: Output file\r\n doc: Write report to filename.\r\n type: string\r\n default: bandit_report.json\r\n output_format:\r\n label: Output format\r\n doc: Specify output format.\r\n type: string\r\n default: json\r\n source_directory: Directory\r\n verbose: boolean\r\n\r\n outputs:\r\n bandit_report:\r\n type: File\r\n outputBinding:\r\n glob: $(inputs.output_file)\r\n\r\n baseCommand: sh\r\n arguments:\r\n - script.sh\r\n id: bandit_tool", - "version": "0.1" - } - }, - { - "model": "backend.commandlinetool", - "pk": "clone", - "fields": { - "name": "Clone repo", - "definition": "- class: CommandLineTool\r\n\r\n requirements:\r\n DockerRequirement:\r\n dockerPull: alpine/git\r\n InitialWorkDirRequirement:\r\n listing:\r\n - entryname: clone_branch.sh\r\n entry: |-\r\n set -e\r\n\r\n if [ $(inputs.repo_branch) ]; then\r\n echo 'Branch specified: $(inputs.repo_branch). Cloning branch...'\r\n git clone $(inputs.repo_url) -b $(inputs.repo_branch)\r\n else\r\n echo 'No branch specified. Cloning default branch...'\r\n git clone $(inputs.repo_url)\r\n fi\r\n InlineJavascriptRequirement: {}\r\n\r\n inputs:\r\n repo_branch: string\r\n repo_url: string\r\n\r\n outputs:\r\n repo_directory:\r\n type: Directory\r\n outputBinding:\r\n glob: $(inputs.repo_url.split('/').pop().replace('.git',''))\r\n\r\n baseCommand: sh\r\n arguments:\r\n - clone_branch.sh\r\n id: clone_tool", - "version": "0.1" - } - }, - { - "model": "backend.commandlinetool", - "pk": "filter", - "fields": { - "name": "Filter", - "definition": "- class: CommandLineTool\r\n\r\n requirements:\r\n DockerRequirement:\r\n dockerPull: alpine:latest\r\n InitialWorkDirRequirement:\r\n listing:\r\n - entryname: script.sh\r\n entry: |-\r\n cd $(inputs.source_directory.path)\r\n find . -type f -regex \"$(inputs.regex)\" > $HOME/filter.out\r\n InlineJavascriptRequirement: {}\r\n\r\n inputs:\r\n regex: string\r\n source_directory: Directory\r\n\r\n outputs:\r\n file_list:\r\n type: string[]\r\n outputBinding:\r\n glob: filter.out\r\n outputEval: |-\r\n $(self[0].contents.split('\\n').filter(function(line) {return line.trim() !== '';}))\r\n loadContents: true\r\n\r\n baseCommand: sh\r\n arguments:\r\n - script.sh\r\n id: filter_tool", - "version": "0.1" - } - }, - { - "model": "backend.commandlinetool", - "pk": "flake8", - "fields": { - "name": "Flake8", - "definition": "- class: CommandLineTool\r\n\r\n requirements:\r\n DockerRequirement:\r\n dockerPull: eoepca/appquality-flake8-json:v0.1.0\r\n InitialWorkDirRequirement:\r\n listing:\r\n - entryname: script.sh\r\n entry: |-\r\n cd $(inputs.source_directory.path)\r\n\r\n PARAMS=\"--format=$(inputs.output_format) --output-file=$HOME/$(inputs.output_file)\"\r\n if [ \"$(inputs.exit_zero)\" == \"true\" ] ; then\r\n PARAMS=\"$PARAMS --exit-zero\"\r\n fi\r\n if [ \"$(inputs.verbose)\" == \"true\" ] ; then\r\n PARAMS=\"$PARAMS -v\"\r\n fi\r\n\r\n flake8 $PARAMS $(inputs.file_list.join(\" \"))\r\n InlineJavascriptRequirement: {}\r\n\r\n inputs:\r\n exit_zero:\r\n label: Exit with zero\r\n doc: Force Flake8 to use the exit status code 0 even if there are errors.\r\n type: boolean\r\n default: true\r\n file_list: string[]\r\n output_file:\r\n label: Output file\r\n doc: Redirect all output to the specified file.\r\n type: string\r\n default: flake8_report.json\r\n output_format:\r\n label: Output format\r\n doc: Select the formatter used to display errors to the user.\r\n type: string\r\n default: json\r\n source_directory: Directory\r\n verbose: boolean\r\n\r\n outputs:\r\n flake8_report:\r\n type: File\r\n outputBinding:\r\n glob: $(inputs.output_file)\r\n\r\n baseCommand: sh\r\n arguments:\r\n - script.sh\r\n id: flake8_tool", - "version": "0.1" - } - }, - { - "model": "backend.commandlinetool", - "pk": "notebook-bp-validator", - "fields": { - "name": "Jupyter Notebook Best Practices Validator", - "definition": "- class: CommandLineTool\r\n\r\n requirements:\r\n DockerRequirement:\r\n dockerPull: eoepca/notebook-bp-validator:2025-05-12.1\r\n InitialWorkDirRequirement:\r\n listing:\r\n - entryname: script.sh\r\n entry: |-\r\n cd $(inputs.source_directory.path)\r\n\r\n PARAMS=\"-s $(inputs.schema)\"\r\n if [ '$(inputs.abspath)' == \"true\" ] ; then\r\n PARAMS=\"$PARAMS -p\"\r\n fi\r\n\r\n nb-validator $PARAMS $(inputs.file_list.join(\" \")) > ~/notebook-bp-validator_report.json\r\n InlineJavascriptRequirement: {}\r\n\r\n inputs:\r\n abspath: boolean\r\n file_list: string[]\r\n schema: string\r\n source_directory: Directory\r\n\r\n outputs:\r\n notebook-bp-validator_report:\r\n type: File\r\n outputBinding:\r\n glob: notebook-bp-validator_report.json\r\n\r\n baseCommand: sh\r\n arguments:\r\n - script.sh\r\n id: notebook-bp-validator_tool", - "version": "0.1" - } - }, - { - "model": "backend.commandlinetool", - "pk": "papermill", - "fields": { - "name": "Papermill", - "definition": "- class: CommandLineTool\r\n\r\n requirements:\r\n DockerRequirement:\r\n dockerPull: continuumio/miniconda3\r\n InitialWorkDirRequirement:\r\n listing:\r\n - entryname: script.sh\r\n entry: |-\r\n set -e\r\n\r\n cat ~/err.ipynb > ~/out.ipynb\r\n cd $(inputs.source_directory.path)\r\n\r\n pip install pyyaml\r\n\r\n python ~/script.py $(inputs.notebook_path) | tee ~/environment.yml\r\n\r\n if [ ! -s ~/environment.yml ]; then\r\n echo \"Error: environment.yml is empty. Aborting environment creation.\"\r\n exit 1\r\n fi\r\n\r\n ENV=`grep \"name: \" ~/environment.yml | cut - -d' ' -f2`\r\n\r\n conda env create -f ~/environment.yml\r\n conda run -n $ENV pip install ipykernel papermill\r\n conda run -n $ENV ipython kernel install --user --name $ENV \r\n conda run -n $ENV papermill $(inputs.notebook_path) ~/out.ipynb\r\n - entryname: script.py\r\n entry: |-\r\n import json\r\n import yaml\r\n import sys\r\n\r\n try:\r\n with open('$(inputs.notebook_path)') as f:\r\n nb_json = json.load(f)\r\n except Exception as e:\r\n sys.stderr.write(f\"Failed to load notebook: {e}\\\\n\")\r\n sys.exit(1)\r\n\r\n try:\r\n env = nb_json[\"metadata\"][\"software_requirements\"][\"conda_environment\"]\r\n kernelspec = nb_json[\"metadata\"][\"kernelspec\"][\"name\"]\r\n except KeyError as e:\r\n sys.stderr.write(f\"Missing key in notebook metadata: {e}\\\\n\")\r\n sys.exit(1)\r\n\r\n if env[\"name\"] == kernelspec:\r\n if env.get(\"dependencies\"):\r\n res = yaml.dump({\"name\": env[\"name\"], \"dependencies\": env[\"dependencies\"]})\r\n print(res)\r\n else:\r\n sys.stderr.write(\"No dependencies found in the environment specification.\\\\n\")\r\n sys.exit(1)\r\n else:\r\n sys.stderr.write(\"Kernel name does not match the conda environment name.\\\\n\")\r\n sys.exit(1)\r\n - entryname: err.ipynb\r\n entry: |-\r\n {\r\n \"cells\": [\r\n {\r\n \"cell_type\": \"markdown\",\r\n \"metadata\": {},\r\n \"source\": [\r\n \"Error: The notebook could not be run.\"\r\n ]\r\n }\r\n ],\r\n \"metadata\": {\r\n \"language_info\": {\r\n \"name\": \"python\"\r\n }\r\n },\r\n \"nbformat\": 4,\r\n \"nbformat_minor\": 2\r\n }\r\n\r\n inputs:\r\n notebook_path: string\r\n source_directory: Directory\r\n\r\n outputs:\r\n output_nb:\r\n type: File\r\n outputBinding:\r\n glob: out.ipynb\r\n\r\n baseCommand: sh\r\n arguments:\r\n - script.sh\r\n id: papermill_tool", - "version": "0.1" - } - }, - { - "model": "backend.commandlinetool", - "pk": "pylint", - "fields": { - "name": "Pylint", - "definition": "- class: CommandLineTool\r\n\r\n requirements:\r\n DockerRequirement:\r\n dockerPull: cytopia/pylint\r\n InitialWorkDirRequirement:\r\n listing:\r\n - entryname: script.sh\r\n entry: |-\r\n cd $(inputs.source_directory.path)\r\n\r\n PARAMS=\"--output-format=$(inputs.output_format) --output=$HOME/$(inputs.output_file) --disable=$(inputs.disable)\"\r\n if [ \"$(inputs.exit_zero)\" == \"true\" ] ; then\r\n PARAMS=\"$PARAMS --exit-zero\"\r\n fi\r\n if [ \"$(inputs.errors_only)\" == \"true\" ] ; then\r\n PARAMS=\"$PARAMS -E\"\r\n fi\r\n if [ \"$(inputs.verbose)\" == \"true\" ] ; then\r\n PARAMS=\"$PARAMS -v\"\r\n fi\r\n\r\n pylint $PARAMS $(inputs.file_list.join(\" \"))\r\n InlineJavascriptRequirement: {}\r\n\r\n inputs:\r\n disable: string\r\n errors_only: boolean\r\n exit_zero:\r\n doc: |-\r\n Always return a 0 (non-error) status code, even if lint errors are found. This is primarily useful in continuous integration scripts.\r\n type: boolean\r\n default: true\r\n file_list: string[]\r\n output_file:\r\n doc: Specify an output file.\r\n type: string\r\n default: pylint_report.json\r\n output_format:\r\n doc: |-\r\n Set the output format. Available formats are: text, parseable, colorized, json2 (improved json format), json (old json format) and msvs (visual studio). You can also give a reporter class, e.g. mypackage.mymodule.MyReporterClass.\r\n type: string\r\n default: json\r\n source_directory: Directory\r\n verbose: boolean\r\n\r\n outputs:\r\n pylint_report:\r\n type: File\r\n outputBinding:\r\n glob: $(inputs.output_file)\r\n\r\n baseCommand: sh\r\n arguments:\r\n - script.sh\r\n id: pylint_tool", - "version": "0.1" - } - }, - { - "model": "backend.commandlinetool", - "pk": "ruff", - "fields": { - "name": "Ruff", - "definition": "- class: CommandLineTool\r\n\r\n requirements:\r\n DockerRequirement:\r\n dockerPull: ghcr.io/astral-sh/ruff:alpine\r\n InitialWorkDirRequirement:\r\n listing:\r\n - entryname: script.sh\r\n entry: |-\r\n cd $(inputs.source_directory.path)\r\n\r\n PARAMS=\"--output-format $(inputs.output_format) -o $HOME/$(inputs.output_file)\"\r\n if [ \"$(inputs.exit_zero)\" == \"true\" ] ; then\r\n PARAMS=\"$PARAMS -e\"\r\n fi\r\n if [ \"$(inputs.no_cache)\" == \"true\" ] ; then\r\n PARAMS=\"$PARAMS -n\"\r\n fi\r\n if [ \"$(inputs.verbose)\" == \"true\" ] ; then\r\n PARAMS=\"$PARAMS -v\"\r\n fi\r\n\r\n ruff check $PARAMS $(inputs.file_list.join(\" \"))\r\n InlineJavascriptRequirement: {}\r\n\r\n inputs:\r\n exit_zero:\r\n label: Exit with zero\r\n doc: Exit with status code \"0\", even upon detecting lint violations.\r\n type: boolean\r\n default: true\r\n no_cache:\r\n label: Disable cache\r\n doc: Disable cache reads.\r\n type: boolean\r\n default: true\r\n file_list: string[]\r\n output_file:\r\n label: Output file\r\n doc: Specify file to write the linter output to.\r\n type: string\r\n default: ruff_report.json\r\n output_format:\r\n label: Output format\r\n doc: |-\r\n Output serialization format for violations. Possible values: concise, full, json, json-lines, junit, grouped, github, gitlab, pylint, rdjson, azure, sarif.\r\n type: string\r\n default: json\r\n source_directory: Directory\r\n verbose: boolean\r\n\r\n outputs:\r\n ruff_report:\r\n type: File\r\n outputBinding:\r\n glob: $(inputs.output_file)\r\n\r\n baseCommand: sh\r\n arguments:\r\n - script.sh\r\n id: ruff_tool", - "version": "0.1" - } - }, - { - "model": "backend.commandlinetool", - "pk": "save", - "fields": { - "name": "Save report", - "definition": "- class: CommandLineTool\r\n\r\n requirements:\r\n DockerRequirement:\r\n dockerPull: curlimages/curl\r\n InlineJavascriptRequirement: {}\r\n\r\n inputs:\r\n name: string\r\n pipeline_id: string\r\n report: File\r\n run_id: string\r\n server_url: string\r\n\r\n outputs: []\r\n\r\n baseCommand: curl\r\n arguments:\r\n - prefix: -X\r\n valueFrom: POST\r\n - prefix: -L\r\n valueFrom: |-\r\n $('http://' + inputs.server_url + '/api/pipelines/' + inputs.pipeline_id + '/runs/' + inputs.run_id + '/jobreports/?name=' + inputs.name)\r\n - prefix: -H\r\n valueFrom: Content-Type:application/json\r\n - prefix: -d\r\n valueFrom: $('@' + inputs.report.path)\r\n id: save_tool", - "version": "0.1" - } - }, - { - "model": "backend.commandlinetool", - "pk": "sonarqube_create_project", - "fields": { - "name": "[SonarQube] Create project", - "definition": "- class: CommandLineTool\r\n\r\n requirements:\r\n DockerRequirement:\r\n dockerPull: curlimages/curl\r\n InlineJavascriptRequirement: {}\r\n\r\n inputs:\r\n sonarqube_project_key:\r\n type: string\r\n sonarqube_project_name:\r\n type: string\r\n sonarqube_server:\r\n type: string\r\n default: sonarqube-sonarqube.sonarqube:9000\r\n sonarqube_token:\r\n type: string\r\n\r\n outputs:\r\n sonarqube_project_key:\r\n type: string\r\n outputBinding:\r\n outputEval: $(inputs.sonarqube_project_key)\r\n sonarqube_server:\r\n type: string\r\n outputBinding:\r\n outputEval: $(inputs.sonarqube_server)\r\n sonarqube_token:\r\n type: string\r\n outputBinding:\r\n outputEval: $(inputs.sonarqube_token)\r\n\r\n baseCommand:\r\n - curl\r\n arguments:\r\n - prefix: -X\r\n valueFrom: POST\r\n - prefix: -L\r\n valueFrom: $('http://' + inputs.sonarqube_server + '/api/projects/create')\r\n - prefix: -u\r\n valueFrom: $(inputs.sonarqube_token + ':')\r\n - prefix: -d\r\n valueFrom: $('name=' + inputs.sonarqube_project_name)\r\n - prefix: -d\r\n valueFrom: $('project=' + inputs.sonarqube_project_key)\r\n id: sonarqube_create_project_tool", - "version": "0.1" - } - }, - { - "model": "backend.commandlinetool", - "pk": "sonarqube_get_report", - "fields": { - "name": "[SonarQube] Get report", - "definition": "- class: CommandLineTool\r\n\r\n requirements:\r\n DockerRequirement:\r\n dockerPull: curlimages/curl\r\n InlineJavascriptRequirement: {}\r\n\r\n inputs:\r\n sonarqube_project_key:\r\n type: string\r\n sonarqube_server:\r\n type: string\r\n default: sonarqube-sonarqube.sonarqube:9000\r\n sonarqube_token:\r\n type: string\r\n\r\n outputs:\r\n sonarqube_project_key:\r\n type: string\r\n outputBinding:\r\n outputEval: $(inputs.sonarqube_project_key)\r\n sonarqube_report:\r\n type: File\r\n outputBinding:\r\n glob: sonarqube_report.json\r\n sonarqube_server:\r\n type: string\r\n outputBinding:\r\n outputEval: $(inputs.sonarqube_server)\r\n sonarqube_token:\r\n type: string\r\n outputBinding:\r\n outputEval: $(inputs.sonarqube_token)\r\n stdout: sonarqube_report.json\r\n\r\n baseCommand:\r\n - curl\r\n arguments:\r\n - prefix: -L\r\n valueFrom: |-\r\n $('http://' + inputs.sonarqube_server + '/api/issues/search?components=' + inputs.sonarqube_project_key)\r\n - prefix: -u\r\n valueFrom: $(inputs.sonarqube_token + ':')\r\n id: sonarqube_get_report_tool", - "version": "0.1" - } - }, - { - "model": "backend.commandlinetool", - "pk": "sonarqube_scan", - "fields": { - "name": "[SonarQube] Scan repo", - "definition": "- class: CommandLineTool\r\n\r\n requirements:\r\n DockerRequirement:\r\n dockerPull: sonarsource/sonar-scanner-cli\r\n EnvVarRequirement:\r\n envDef:\r\n SONAR_HOST_URL: $('http://' + inputs.sonarqube_server)\r\n SONAR_TOKEN: $(inputs.sonarqube_token)\r\n InlineJavascriptRequirement: {}\r\n\r\n inputs:\r\n sonarqube_project_key:\r\n type: string\r\n sonarqube_server:\r\n type: string\r\n default: sonarqube-sonarqube.sonarqube:9000\r\n sonarqube_token:\r\n type: string\r\n source_directory:\r\n type: Directory\r\n\r\n outputs:\r\n sonarqube_project_key:\r\n type: string\r\n outputBinding:\r\n glob:\r\n outputEval: $(inputs.sonarqube_project_key)\r\n sonarqube_server:\r\n type: string\r\n outputBinding:\r\n glob:\r\n outputEval: $(inputs.sonarqube_server)\r\n sonarqube_token:\r\n type: string\r\n outputBinding:\r\n glob:\r\n outputEval: $(inputs.sonarqube_token)\r\n\r\n baseCommand:\r\n - sonar-scanner\r\n arguments:\r\n - prefix: -D\r\n valueFrom: $('sonar.projectKey=' + inputs.sonarqube_project_key)\r\n separate: false\r\n # - prefix: -D\r\n # valueFrom: $('sonar.userHome=$HOME')\r\n # separate: false\r\n - prefix: -D\r\n valueFrom: $('sonar.projectBaseDir=' + inputs.source_directory.path + '/../')\r\n separate: false\r\n # - prefix: -D\r\n # valueFrom: $('sonar.source=$HOME' /*+ inputs.source_directory.path*/)\r\n # separate: false\r\n - prefix: -X\r\n id: sonarqube_scan_tool", - "version": "0.1" - } - }, - { - "model": "backend.commandlinetool", - "pk": "trivy", - "fields": { - "name": "Trivy", - "definition": "- class: CommandLineTool\r\n\r\n requirements:\r\n DockerRequirement:\r\n dockerPull: aquasec/trivy:latest\r\n\r\n inputs:\r\n image: string\r\n\r\n outputs:\r\n trivy_report:\r\n type: File\r\n outputBinding:\r\n glob: trivy_report.json\r\n\r\n baseCommand: trivy\r\n arguments:\r\n - image\r\n - prefix: -f\r\n valueFrom: json\r\n - prefix: -o\r\n valueFrom: trivy_report.json\r\n - $(inputs.image)\r\n id: trivy_tool", - "version": "0.1" - } - } -] \ No newline at end of file diff --git a/backend/backend/fixtures/backend.json b/backend/backend/fixtures/backend.json new file mode 100644 index 0000000..eb48f04 --- /dev/null +++ b/backend/backend/fixtures/backend.json @@ -0,0 +1,716 @@ +[ + { + "model": "backend.tag", + "pk": 1, + "fields": { + "name": "asset: python" + } + }, + { + "model": "backend.tag", + "pk": 2, + "fields": { + "name": "asset: other" + } + }, + { + "model": "backend.tag", + "pk": 3, + "fields": { + "name": "asset: cwl" + } + }, + { + "model": "backend.tag", + "pk": 4, + "fields": { + "name": "asset: notebook" + } + }, + { + "model": "backend.tag", + "pk": 5, + "fields": { + "name": "type: best practice" + } + }, + { + "model": "backend.tag", + "pk": 6, + "fields": { + "name": "type: app quality" + } + }, + { + "model": "backend.tag", + "pk": 7, + "fields": { + "name": "type: app performance" + } + }, + { + "model": "backend.tag", + "pk": 8, + "fields": { + "name": "type: init" + } + }, + { + "model": "backend.tag", + "pk": 9, + "fields": { + "name": "asset: docker" + } + }, + { + "model": "backend.commandlinetool", + "pk": "ap_validator", + "fields": { + "name": "Application Package Validator", + "definition": "- class: CommandLineTool\r\n\r\n requirements:\r\n DockerRequirement:\r\n dockerPull: nexus.spaceapplications.com/repository/docker-eoepca/ap_validator:2025-03-05.1\r\n InitialWorkDirRequirement:\r\n listing:\r\n - entryname: script.sh\r\n entry: |-\r\n cd $(inputs.source_directory.path)\r\n\r\n ap-validator \\\\\r\n --format json \\\\\r\n --detail '$(inputs.detail)' \\\\\r\n --entry-point '$(inputs.entry_point)' \\\\\r\n '$(inputs.file_path)' > ~/ap_validator_report.json\r\n\r\n exit 0\r\n InlineJavascriptRequirement: {}\r\n\r\n inputs:\r\n file_path: string\r\n source_directory: Directory\r\n detail: string\r\n entry_point: string\r\n\r\n outputs:\r\n ap_validator_report:\r\n type: File\r\n outputBinding:\r\n glob: ap_validator_report.json\r\n\r\n baseCommand: sh\r\n arguments:\r\n - script.sh\r\n id: ap_validator_tool", + "version": "0.1" + } + }, + { + "model": "backend.commandlinetool", + "pk": "bandit", + "fields": { + "name": "Bandit", + "definition": "- class: CommandLineTool\r\n\r\n requirements:\r\n DockerRequirement:\r\n dockerPull: cytopia/bandit\r\n InitialWorkDirRequirement:\r\n listing:\r\n - entryname: script.sh\r\n entry: |-\r\n cd $(inputs.source_directory.path)\r\n\r\n PARAMS=\"-f $(inputs.output_format) -o $HOME/$(inputs.output_file)\"\r\n if [ \"$(inputs.exit_zero)\" == \"true\" ] ; then\r\n PARAMS=\"$PARAMS --exit-zero\"\r\n fi\r\n if [ \"$(inputs.verbose)\" == \"true\" ] ; then\r\n PARAMS=\"$PARAMS -v\"\r\n fi\r\n\r\n bandit $PARAMS $(inputs.file_list.join(\" \"))\r\n InlineJavascriptRequirement: {}\r\n\r\n inputs:\r\n exit_zero:\r\n label: Exit with zero\r\n doc: Exit with 0, even with results found.\r\n type: boolean\r\n default: true\r\n file_list: string[]\r\n output_file:\r\n label: Output file\r\n doc: Write report to filename.\r\n type: string\r\n default: bandit_report.json\r\n output_format:\r\n label: Output format\r\n doc: Specify output format.\r\n type: string\r\n default: json\r\n source_directory: Directory\r\n verbose: boolean\r\n\r\n outputs:\r\n bandit_report:\r\n type: File\r\n outputBinding:\r\n glob: $(inputs.output_file)\r\n\r\n baseCommand: sh\r\n arguments:\r\n - script.sh\r\n id: bandit_tool", + "version": "0.1" + } + }, + { + "model": "backend.commandlinetool", + "pk": "clone", + "fields": { + "name": "Clone repo", + "definition": "- class: CommandLineTool\r\n\r\n requirements:\r\n DockerRequirement:\r\n dockerPull: alpine/git\r\n InitialWorkDirRequirement:\r\n listing:\r\n - entryname: clone_branch.sh\r\n entry: |-\r\n set -e\r\n\r\n if [ $(inputs.repo_branch) ]; then\r\n echo 'Branch specified: $(inputs.repo_branch). Cloning branch...'\r\n git clone $(inputs.repo_url) -b $(inputs.repo_branch)\r\n else\r\n echo 'No branch specified. Cloning default branch...'\r\n git clone $(inputs.repo_url)\r\n fi\r\n InlineJavascriptRequirement: {}\r\n\r\n inputs:\r\n repo_branch: string\r\n repo_url: string\r\n\r\n outputs:\r\n repo_directory:\r\n type: Directory\r\n outputBinding:\r\n glob: $(inputs.repo_url.split('/').pop().replace('.git',''))\r\n\r\n baseCommand: sh\r\n arguments:\r\n - clone_branch.sh\r\n id: clone_tool", + "version": "0.1" + } + }, + { + "model": "backend.commandlinetool", + "pk": "filter", + "fields": { + "name": "Filter", + "definition": "- class: CommandLineTool\r\n\r\n requirements:\r\n DockerRequirement:\r\n dockerPull: alpine:latest\r\n InitialWorkDirRequirement:\r\n listing:\r\n - entryname: script.sh\r\n entry: |-\r\n cd $(inputs.source_directory.path)\r\n find . -type f -regex \"$(inputs.regex)\" > $HOME/filter.out\r\n InlineJavascriptRequirement: {}\r\n\r\n inputs:\r\n regex: string\r\n source_directory: Directory\r\n\r\n outputs:\r\n file_list:\r\n type: string[]\r\n outputBinding:\r\n glob: filter.out\r\n outputEval: |-\r\n $(self[0].contents.split('\\n').filter(function(line) {return line.trim() !== '';}))\r\n loadContents: true\r\n\r\n baseCommand: sh\r\n arguments:\r\n - script.sh\r\n id: filter_tool", + "version": "0.1" + } + }, + { + "model": "backend.commandlinetool", + "pk": "flake8", + "fields": { + "name": "Flake8", + "definition": "- class: CommandLineTool\r\n\r\n requirements:\r\n DockerRequirement:\r\n dockerPull: eoepca/appquality-flake8-json:v0.1.0\r\n InitialWorkDirRequirement:\r\n listing:\r\n - entryname: script.sh\r\n entry: |-\r\n cd $(inputs.source_directory.path)\r\n\r\n PARAMS=\"--format=$(inputs.output_format) --output-file=$HOME/$(inputs.output_file)\"\r\n if [ \"$(inputs.exit_zero)\" == \"true\" ] ; then\r\n PARAMS=\"$PARAMS --exit-zero\"\r\n fi\r\n if [ \"$(inputs.verbose)\" == \"true\" ] ; then\r\n PARAMS=\"$PARAMS -v\"\r\n fi\r\n\r\n flake8 $PARAMS $(inputs.file_list.join(\" \"))\r\n InlineJavascriptRequirement: {}\r\n\r\n inputs:\r\n exit_zero:\r\n label: Exit with zero\r\n doc: Force Flake8 to use the exit status code 0 even if there are errors.\r\n type: boolean\r\n default: true\r\n file_list: string[]\r\n output_file:\r\n label: Output file\r\n doc: Redirect all output to the specified file.\r\n type: string\r\n default: flake8_report.json\r\n output_format:\r\n label: Output format\r\n doc: Select the formatter used to display errors to the user.\r\n type: string\r\n default: json\r\n source_directory: Directory\r\n verbose: boolean\r\n\r\n outputs:\r\n flake8_report:\r\n type: File\r\n outputBinding:\r\n glob: $(inputs.output_file)\r\n\r\n baseCommand: sh\r\n arguments:\r\n - script.sh\r\n id: flake8_tool", + "version": "0.1" + } + }, + { + "model": "backend.commandlinetool", + "pk": "notebook-bp-validator", + "fields": { + "name": "Jupyter Notebook Best Practices Validator", + "definition": "- class: CommandLineTool\r\n\r\n requirements:\r\n DockerRequirement:\r\n dockerPull: eoepca/notebook-bp-validator:2025-05-12.1\r\n InitialWorkDirRequirement:\r\n listing:\r\n - entryname: script.sh\r\n entry: |-\r\n cd $(inputs.source_directory.path)\r\n\r\n PARAMS=\"-s $(inputs.schema)\"\r\n if [ '$(inputs.abspath)' == \"true\" ] ; then\r\n PARAMS=\"$PARAMS -p\"\r\n fi\r\n\r\n nb-validator $PARAMS $(inputs.file_list.join(\" \")) > ~/notebook-bp-validator_report.json\r\n InlineJavascriptRequirement: {}\r\n\r\n inputs:\r\n abspath: boolean\r\n file_list: string[]\r\n schema: string\r\n source_directory: Directory\r\n\r\n outputs:\r\n notebook-bp-validator_report:\r\n type: File\r\n outputBinding:\r\n glob: notebook-bp-validator_report.json\r\n\r\n baseCommand: sh\r\n arguments:\r\n - script.sh\r\n id: notebook-bp-validator_tool", + "version": "0.1" + } + }, + { + "model": "backend.commandlinetool", + "pk": "papermill", + "fields": { + "name": "Papermill", + "definition": "- class: CommandLineTool\r\n\r\n requirements:\r\n DockerRequirement:\r\n dockerPull: continuumio/miniconda3\r\n InitialWorkDirRequirement:\r\n listing:\r\n - entryname: script.sh\r\n entry: |-\r\n set -e\r\n\r\n cat ~/err.ipynb > ~/out.ipynb\r\n echo \"[Papermill] Source directory: $(inputs.source_directory.path)\"\r\n echo \"[Papermill] Notebook path: $(inputs.notebook_path)\"\r\n echo \"[Papermill] Extract requirements: $(inputs.extract_requirements)\"\r\n echo \"[Papermill] Extra requirements: $(inputs.extra_requirements)\"\r\n echo \"[Papermill] Execution parameters: $(inputs.execution_parameters)\"\r\n\r\n cd $(inputs.source_directory.path)\r\n\r\n if [ -s environment.yml ]; then\r\n echo \"Info: Found non-empty environment.yml file in the source folder. Will use it to setup the Conda environment.\"\r\n ENV=`grep \"name: \" environment.yml | cut - -d' ' -f2`\r\n echo \"Info: Environment name: $ENV\"\r\n cp -p environment.yml ~/environment.yml\r\n touch ~/requirements.txt\r\n else\r\n if [ \"$(inputs.extract_requirements)\" = \"true\" ]; then\r\n echo \"Info: Generating requirements.txt\"\r\n cd ~\r\n apt update && apt install -y jq\r\n pip install pipreqsnb\r\n pipreqsnb --force --encoding utf-8 --savepath ~/requirements.txt $(inputs.source_directory.path)/$(inputs.notebook_path)\r\n echo \"Info: Generating environment.yml\"\r\n ENV=`cat $(inputs.source_directory.path)/$(inputs.notebook_path) | jq .metadata.kernelspec.name -r`\r\n echo \"Info: Environment name: $ENV\"\r\n echo \"name: $ENV\" > ~/environment.yml\r\n else\r\n echo \"Info: Generating environment.yml\"\r\n pip install pyyaml\r\n python ~/script.py $(inputs.notebook_path) | tee ~/environment.yml\r\n touch ~/requirements.txt\r\n fi\r\n fi\r\n\r\n if [ ! -s ~/environment.yml ]; then\r\n echo \"Error: environment.yml is empty. Aborting environment creation.\"\r\n exit 1\r\n fi\r\n\r\n echo \"Requirements file:\"\r\n cat ~/requirements.txt\r\n\r\n echo \"Environment file:\"\r\n cat ~/environment.yml\r\n\r\n ENV=`grep \"name: \" ~/environment.yml | cut - -d' ' -f2`\r\n echo \"Info: Environment name: $ENV\"\r\n\r\n conda env create -f ~/environment.yml\r\n conda run -n $ENV pip install ipykernel papermill\r\n conda run -n $ENV pip install -r ~/requirements.txt\r\n if [ \"$(inputs.extra_requirements)\" != \"\" ]; then\r\n conda run -n $ENV pip install $(inputs.extra_requirements)\r\n fi\r\n conda run -n $ENV ipython kernel install --user --name $ENV\r\n conda run -n $ENV papermill $(inputs.source_directory.path)/$(inputs.notebook_path) ~/out.ipynb\r\n\r\n - entryname: script.py\r\n entry: |-\r\n import json\r\n import yaml\r\n import sys\r\n\r\n try:\r\n with open('$(inputs.notebook_path)') as f:\r\n nb_json = json.load(f)\r\n except Exception as e:\r\n sys.stderr.write(f\"Failed to load notebook: {e}\\n\")\r\n sys.exit(1)\r\n\r\n conda_env = nb_json.get(\"metadata\", {}).get(\"software_requirements\", {}).get(\"conda_environment\", None)\r\n\r\n try:\r\n kernel_name = nb_json[\"metadata\"][\"kernelspec\"][\"name\"]\r\n except KeyError as e:\r\n sys.stderr.write(f\"Missing key in notebook metadata: {e}\\n\")\r\n sys.exit(1)\r\n\r\n if conda_env:\r\n if conda_env.get(\"name\", None) == kernel_name:\r\n if conda_env.get(\"dependencies\"):\r\n res = yaml.dump({\"name\": conda_env[\"name\"], \"dependencies\": conda_env[\"dependencies\"]})\r\n print(res)\r\n else:\r\n sys.stderr.write(\"No dependencies found in the environment specification.\\n\")\r\n sys.exit(1)\r\n else:\r\n sys.stderr.write(\"Kernel name does not match the conda environment name.\\n\")\r\n sys.exit(1)\r\n else:\r\n res = yaml.dump({\"name\": kernel_name, \"dependencies\": []})\r\n print(res)\r\n\r\n - entryname: err.ipynb\r\n entry: |-\r\n {\r\n \"cells\": [\r\n {\r\n \"cell_type\": \"markdown\",\r\n \"metadata\": {},\r\n \"source\": [\r\n \"Error: The notebook could not be run.\"\r\n ]\r\n }\r\n ],\r\n \"metadata\": {\r\n \"language_info\": {\r\n \"name\": \"python\"\r\n }\r\n },\r\n \"nbformat\": 4,\r\n \"nbformat_minor\": 2\r\n }\r\n\r\n inputs:\r\n notebook_path: string\r\n extra_requirements:\r\n label: Python requirements\r\n doc: Space-separated list of Python libraries to be installed before executing the notebook\r\n type: string\r\n default: \"\"\r\n extract_requirements:\r\n label: Extract requirements\r\n doc: Inspect the notebook cells to derive library requirements\r\n type: boolean\r\n default: false\r\n execution_parameters:\r\n label: Execution parameters\r\n doc: These parameters are provided as inputs to the executed notebooks\r\n type: string\r\n default: \"\"\r\n source_directory: Directory\r\n\r\n outputs:\r\n output_nb:\r\n type: File\r\n outputBinding:\r\n glob: out.ipynb\r\n\r\n baseCommand: sh\r\n arguments:\r\n - script.sh\r\n id: papermill_tool", + "version": "0.1" + } + }, + { + "model": "backend.commandlinetool", + "pk": "pylint", + "fields": { + "name": "Pylint", + "definition": "- class: CommandLineTool\r\n\r\n requirements:\r\n DockerRequirement:\r\n dockerPull: cytopia/pylint\r\n InitialWorkDirRequirement:\r\n listing:\r\n - entryname: script.sh\r\n entry: |-\r\n cd $(inputs.source_directory.path)\r\n\r\n PARAMS=\"--output-format=$(inputs.output_format) --output=$HOME/$(inputs.output_file) --disable=$(inputs.disable)\"\r\n if [ \"$(inputs.exit_zero)\" == \"true\" ] ; then\r\n PARAMS=\"$PARAMS --exit-zero\"\r\n fi\r\n if [ \"$(inputs.errors_only)\" == \"true\" ] ; then\r\n PARAMS=\"$PARAMS -E\"\r\n fi\r\n if [ \"$(inputs.verbose)\" == \"true\" ] ; then\r\n PARAMS=\"$PARAMS -v\"\r\n fi\r\n\r\n pylint $PARAMS $(inputs.file_list.join(\" \"))\r\n InlineJavascriptRequirement: {}\r\n\r\n inputs:\r\n disable: string\r\n errors_only: boolean\r\n exit_zero:\r\n doc: |-\r\n Always return a 0 (non-error) status code, even if lint errors are found. This is primarily useful in continuous integration scripts.\r\n type: boolean\r\n default: true\r\n file_list: string[]\r\n output_file:\r\n doc: Specify an output file.\r\n type: string\r\n default: pylint_report.json\r\n output_format:\r\n doc: |-\r\n Set the output format. Available formats are: text, parseable, colorized, json2 (improved json format), json (old json format) and msvs (visual studio). You can also give a reporter class, e.g. mypackage.mymodule.MyReporterClass.\r\n type: string\r\n default: json\r\n source_directory: Directory\r\n verbose: boolean\r\n\r\n outputs:\r\n pylint_report:\r\n type: File\r\n outputBinding:\r\n glob: $(inputs.output_file)\r\n\r\n baseCommand: sh\r\n arguments:\r\n - script.sh\r\n id: pylint_tool", + "version": "0.1" + } + }, + { + "model": "backend.commandlinetool", + "pk": "ruff", + "fields": { + "name": "Ruff", + "definition": "- class: CommandLineTool\r\n\r\n requirements:\r\n DockerRequirement:\r\n dockerPull: ghcr.io/astral-sh/ruff:alpine\r\n InitialWorkDirRequirement:\r\n listing:\r\n - entryname: script.sh\r\n entry: |-\r\n cd $(inputs.source_directory.path)\r\n\r\n PARAMS=\"--output-format $(inputs.output_format) -o $HOME/$(inputs.output_file)\"\r\n if [ \"$(inputs.exit_zero)\" == \"true\" ] ; then\r\n PARAMS=\"$PARAMS -e\"\r\n fi\r\n if [ \"$(inputs.no_cache)\" == \"true\" ] ; then\r\n PARAMS=\"$PARAMS -n\"\r\n fi\r\n if [ \"$(inputs.verbose)\" == \"true\" ] ; then\r\n PARAMS=\"$PARAMS -v\"\r\n fi\r\n\r\n ruff check $PARAMS $(inputs.file_list.join(\" \"))\r\n InlineJavascriptRequirement: {}\r\n\r\n inputs:\r\n exit_zero:\r\n label: Exit with zero\r\n doc: Exit with status code \"0\", even upon detecting lint violations.\r\n type: boolean\r\n default: true\r\n no_cache:\r\n label: Disable cache\r\n doc: Disable cache reads.\r\n type: boolean\r\n default: true\r\n file_list: string[]\r\n output_file:\r\n label: Output file\r\n doc: Specify file to write the linter output to.\r\n type: string\r\n default: ruff_report.json\r\n output_format:\r\n label: Output format\r\n doc: |-\r\n Output serialization format for violations. Possible values: concise, full, json, json-lines, junit, grouped, github, gitlab, pylint, rdjson, azure, sarif.\r\n type: string\r\n default: json\r\n source_directory: Directory\r\n verbose: boolean\r\n\r\n outputs:\r\n ruff_report:\r\n type: File\r\n outputBinding:\r\n glob: $(inputs.output_file)\r\n\r\n baseCommand: sh\r\n arguments:\r\n - script.sh\r\n id: ruff_tool", + "version": "0.1" + } + }, + { + "model": "backend.commandlinetool", + "pk": "save", + "fields": { + "name": "Save report", + "definition": "- class: CommandLineTool\r\n\r\n requirements:\r\n DockerRequirement:\r\n dockerPull: curlimages/curl\r\n InlineJavascriptRequirement: {}\r\n\r\n inputs:\r\n name: string\r\n instance:\r\n type: string\r\n default: \"\"\r\n pipeline_id: string\r\n report: File\r\n run_id: string\r\n server_url: string\r\n\r\n outputs: []\r\n\r\n baseCommand: curl\r\n arguments:\r\n - prefix: -X\r\n valueFrom: POST\r\n - prefix: -L\r\n valueFrom: |-\r\n $(inputs.server_url + '/api/pipelines/' + inputs.pipeline_id + '/runs/' + inputs.run_id + '/jobreports/?name=' + inputs.name + '&instance=' + inputs.instance)\r\n - prefix: -H\r\n valueFrom: Content-Type:application/json\r\n - prefix: -d\r\n valueFrom: $('@' + inputs.report.path)\r\n id: save_tool", + "version": "0.1" + } + }, + { + "model": "backend.commandlinetool", + "pk": "sonarqube_create_project", + "fields": { + "name": "[SonarQube] Create project", + "definition": "- class: CommandLineTool\r\n\r\n requirements:\r\n DockerRequirement:\r\n dockerPull: curlimages/curl\r\n InlineJavascriptRequirement: {}\r\n\r\n inputs:\r\n sonarqube_project_key:\r\n type: string\r\n sonarqube_project_name:\r\n type: string\r\n sonarqube_server:\r\n type: string\r\n default: sonarqube-sonarqube.sonarqube:9000\r\n sonarqube_token:\r\n type: string\r\n\r\n outputs:\r\n sonarqube_project_key:\r\n type: string\r\n outputBinding:\r\n outputEval: $(inputs.sonarqube_project_key)\r\n sonarqube_server:\r\n type: string\r\n outputBinding:\r\n outputEval: $(inputs.sonarqube_server)\r\n sonarqube_token:\r\n type: string\r\n outputBinding:\r\n outputEval: $(inputs.sonarqube_token)\r\n\r\n baseCommand:\r\n - curl\r\n arguments:\r\n - prefix: -X\r\n valueFrom: POST\r\n - prefix: -L\r\n valueFrom: $('http://' + inputs.sonarqube_server + '/api/projects/create')\r\n - prefix: -u\r\n valueFrom: $(inputs.sonarqube_token + ':')\r\n - prefix: -d\r\n valueFrom: $('name=' + inputs.sonarqube_project_name)\r\n - prefix: -d\r\n valueFrom: $('project=' + inputs.sonarqube_project_key)\r\n id: sonarqube_create_project_tool", + "version": "0.1" + } + }, + { + "model": "backend.commandlinetool", + "pk": "sonarqube_get_report", + "fields": { + "name": "[SonarQube] Get report", + "definition": "- class: CommandLineTool\r\n\r\n requirements:\r\n DockerRequirement:\r\n dockerPull: curlimages/curl\r\n InlineJavascriptRequirement: {}\r\n\r\n inputs:\r\n sonarqube_project_key:\r\n type: string\r\n sonarqube_server:\r\n type: string\r\n default: sonarqube-sonarqube.sonarqube:9000\r\n sonarqube_token:\r\n type: string\r\n\r\n outputs:\r\n sonarqube_project_key:\r\n type: string\r\n outputBinding:\r\n outputEval: $(inputs.sonarqube_project_key)\r\n sonarqube_report:\r\n type: File\r\n outputBinding:\r\n glob: sonarqube_report.json\r\n sonarqube_server:\r\n type: string\r\n outputBinding:\r\n outputEval: $(inputs.sonarqube_server)\r\n sonarqube_token:\r\n type: string\r\n outputBinding:\r\n outputEval: $(inputs.sonarqube_token)\r\n stdout: sonarqube_report.json\r\n\r\n baseCommand:\r\n - curl\r\n arguments:\r\n - prefix: -L\r\n valueFrom: |-\r\n $('http://' + inputs.sonarqube_server + '/api/issues/search?components=' + inputs.sonarqube_project_key)\r\n - prefix: -u\r\n valueFrom: $(inputs.sonarqube_token + ':')\r\n id: sonarqube_get_report_tool", + "version": "0.1" + } + }, + { + "model": "backend.commandlinetool", + "pk": "sonarqube_scan", + "fields": { + "name": "[SonarQube] Scan repo", + "definition": "- class: CommandLineTool\r\n\r\n requirements:\r\n DockerRequirement:\r\n dockerPull: sonarsource/sonar-scanner-cli\r\n EnvVarRequirement:\r\n envDef:\r\n SONAR_HOST_URL: $('http://' + inputs.sonarqube_server)\r\n SONAR_TOKEN: $(inputs.sonarqube_token)\r\n InlineJavascriptRequirement: {}\r\n\r\n inputs:\r\n sonarqube_project_key:\r\n type: string\r\n sonarqube_server:\r\n type: string\r\n default: sonarqube-sonarqube.sonarqube:9000\r\n sonarqube_token:\r\n type: string\r\n source_directory:\r\n type: Directory\r\n\r\n outputs:\r\n sonarqube_project_key:\r\n type: string\r\n outputBinding:\r\n glob:\r\n outputEval: $(inputs.sonarqube_project_key)\r\n sonarqube_server:\r\n type: string\r\n outputBinding:\r\n glob:\r\n outputEval: $(inputs.sonarqube_server)\r\n sonarqube_token:\r\n type: string\r\n outputBinding:\r\n glob:\r\n outputEval: $(inputs.sonarqube_token)\r\n\r\n baseCommand:\r\n - sonar-scanner\r\n arguments:\r\n - prefix: -D\r\n valueFrom: $('sonar.projectKey=' + inputs.sonarqube_project_key)\r\n separate: false\r\n # - prefix: -D\r\n # valueFrom: $('sonar.userHome=$HOME')\r\n # separate: false\r\n - prefix: -D\r\n valueFrom: $('sonar.projectBaseDir=' + inputs.source_directory.path + '/../')\r\n separate: false\r\n # - prefix: -D\r\n # valueFrom: $('sonar.source=$HOME' /*+ inputs.source_directory.path*/)\r\n # separate: false\r\n - prefix: -X\r\n id: sonarqube_scan_tool", + "version": "0.1" + } + }, + { + "model": "backend.commandlinetool", + "pk": "trivy", + "fields": { + "name": "Trivy", + "definition": "- class: CommandLineTool\r\n\r\n requirements:\r\n DockerRequirement:\r\n dockerPull: aquasec/trivy:latest\r\n\r\n inputs:\r\n image: string\r\n\r\n outputs:\r\n trivy_report:\r\n type: File\r\n outputBinding:\r\n glob: trivy_report.json\r\n\r\n baseCommand: trivy\r\n arguments:\r\n - image\r\n - prefix: -f\r\n valueFrom: json\r\n - prefix: -o\r\n valueFrom: trivy_report.json\r\n - $(inputs.image)\r\n id: trivy_tool", + "version": "0.1" + } + }, + { + "model": "backend.subworkflow", + "pk": "ap_validator_subworkflow", + "fields": { + "name": "Application Package Validator", + "description": "Validation tool for checking OGC compliance of CWL files for application packages.", + "pipeline_step": "ap_validator_subworkflow:\r\n in:\r\n ap_validator.detail: ap_validator_subworkflow.ap_validator.detail\r\n ap_validator.entry_point: ap_validator_subworkflow.ap_validator.entry_point\r\n filter.regex: ap_validator_subworkflow.filter.regex\r\n pipeline_id: pipeline_id\r\n run_id: run_id\r\n server_url: server_url\r\n source_directory: clone_step/repo_directory\r\n run: '#ap_validator_subworkflow'\r\n out: []", + "definition": "- class: Workflow\r\n\r\n requirements:\r\n ScatterFeatureRequirement: {}\r\n\r\n inputs:\r\n name:\r\n type: string\r\n default: ap_validator\r\n ap_validator.detail: string\r\n ap_validator.entry_point: string\r\n filter.regex: string\r\n pipeline_id: string\r\n run_id: string\r\n server_url: string\r\n source_directory: Directory\r\n\r\n outputs:\r\n []\r\n\r\n steps:\r\n filter_ap_validator_step:\r\n in:\r\n regex: filter.regex\r\n source_directory: source_directory\r\n run: '#filter_tool'\r\n out:\r\n - file_list\r\n ap_validator_step:\r\n in:\r\n file_path: filter_ap_validator_step/file_list\r\n source_directory: source_directory\r\n detail: ap_validator.detail\r\n entry_point: ap_validator.entry_point\r\n scatter: file_path\r\n run: '#ap_validator_tool'\r\n out:\r\n - ap_validator_report\r\n save_ap_validator_step:\r\n in:\r\n name: name\r\n pipeline_id: pipeline_id\r\n report: ap_validator_step/ap_validator_report\r\n run_id: run_id\r\n server_url: server_url\r\n scatter: report\r\n run: '#save_tool'\r\n out: []\r\n id: ap_validator_subworkflow\r\n{% for tool in tools %}{{ tool.definition }}\r\n{% endfor %}", + "user_params": { + "filter": { + "regex": { + "type": "string", + "label": "regex", + "default": ".*\\.cwl" + } + }, + "ap_validator": { + "detail": { + "doc": "Output detail (none|errors|hints|all). Default: hints", + "type": "string", + "label": "Detail", + "default": "hints" + }, + "entry_point": { + "doc": "Name of entry point (Workflow or CommandLineTool)", + "type": "string", + "label": "Entry point", + "default": "main" + } + } + }, + "version": "0.1", + "tags": [ + 3, + 5 + ], + "tools": [ + "filter", + "ap_validator", + "save" + ] + } + }, + { + "model": "backend.subworkflow", + "pk": "bandit_subworkflow", + "fields": { + "name": "Bandit", + "description": "Bandit - Bandit is a tool designed to find common security issues in Python code", + "pipeline_step": "bandit_subworkflow:\r\n in:\r\n bandit.verbose: bandit_subworkflow.bandit.verbose\r\n filter.regex: bandit_subworkflow.filter.regex\r\n pipeline_id: pipeline_id\r\n run_id: run_id\r\n server_url: server_url\r\n source_directory: clone_step/repo_directory\r\n run: '#bandit_subworkflow'\r\n out: []", + "definition": "- class: Workflow\r\n\r\n inputs:\r\n name:\r\n type: string\r\n default: bandit\r\n bandit.verbose: boolean\r\n filter.regex: string\r\n pipeline_id: string\r\n run_id: string\r\n server_url: string\r\n source_directory: Directory\r\n\r\n outputs:\r\n bandit_report:\r\n type: File\r\n outputSource: bandit_step/bandit_report\r\n\r\n steps:\r\n bandit_step:\r\n in:\r\n file_list: filter_bandit_step/file_list\r\n source_directory: source_directory\r\n verbose: bandit.verbose\r\n run: '#bandit_tool'\r\n out:\r\n - bandit_report\r\n filter_bandit_step:\r\n in:\r\n regex: filter.regex\r\n source_directory: source_directory\r\n run: '#filter_tool'\r\n out:\r\n - file_list\r\n save_bandit_step:\r\n in:\r\n name: name\r\n pipeline_id: pipeline_id\r\n report: bandit_step/bandit_report\r\n run_id: run_id\r\n server_url: server_url\r\n run: '#save_tool'\r\n out: []\r\n id: bandit_subworkflow\r\n{% for tool in tools %}{{ tool.definition }}\r\n{% endfor %}", + "user_params": { + "bandit": { + "verbose": { + "doc": "Output extra information like excluded and included files.", + "type": "boolean", + "label": "Verbose", + "default": false + } + }, + "filter": { + "regex": { + "type": "string", + "label": "regex", + "default": ".*\\.py" + } + } + }, + "version": "0.1", + "tags": [ + 1, + 6 + ], + "tools": [ + "filter", + "bandit", + "save" + ] + } + }, + { + "model": "backend.subworkflow", + "pk": "clone_subworkflow", + "fields": { + "name": "Clone repo", + "description": "git-clone - Clone a repository into a new directory", + "pipeline_step": "clone_step:\r\n in:\r\n clone.repo_branch: clone_subworkflow.clone.repo_branch\r\n clone.repo_url: clone_subworkflow.clone.repo_url\r\n run: '#clone_subworkflow'\r\n out:\r\n - repo_directory", + "definition": "- class: Workflow\r\n\r\n inputs:\r\n clone.repo_branch:\r\n type: string\r\n default: ''\r\n clone.repo_url: string\r\n\r\n outputs:\r\n repo_directory:\r\n type: Directory\r\n outputSource: clone_tool_step/repo_directory\r\n\r\n steps:\r\n clone_tool_step:\r\n in:\r\n repo_branch: clone.repo_branch\r\n repo_url: clone.repo_url\r\n run: '#clone_tool'\r\n out:\r\n - repo_directory\r\n id: clone_subworkflow\r\n{% for tool in tools %}{{ tool.definition }}\r\n{% endfor %}", + "user_params": { + "clone": { + "repo_url": { + "doc": "URL to the repository to clone.", + "type": "string", + "label": "Repo URL", + "default": "" + }, + "repo_branch": { + "doc": "Branch to checkout instead of the remote HEAD.", + "type": "string", + "label": "Repo branch", + "default": "" + } + } + }, + "version": "0.1", + "tags": [ + 2, + 8 + ], + "tools": [ + "clone" + ] + } + }, + { + "model": "backend.subworkflow", + "pk": "flake8_subworkflow", + "fields": { + "name": "Flake8", + "description": "flake8 - Style guide enforcement tool for Python", + "pipeline_step": "flake8_subworkflow:\r\n in:\r\n filter.regex: flake8_subworkflow.filter.regex\r\n flake8.verbose: flake8_subworkflow.flake8.verbose\r\n pipeline_id: pipeline_id\r\n run_id: run_id\r\n server_url: server_url\r\n source_directory: clone_step/repo_directory\r\n run: '#flake8_subworkflow'\r\n out: []", + "definition": "- class: Workflow\r\n\r\n inputs:\r\n name:\r\n type: string\r\n default: flake8\r\n filter.regex: string\r\n flake8.verbose: boolean\r\n pipeline_id: string\r\n run_id: string\r\n server_url: string\r\n source_directory: Directory\r\n\r\n outputs:\r\n flake8_report:\r\n type: File\r\n outputSource: flake8_step/flake8_report\r\n\r\n steps:\r\n filter_flake8_step:\r\n in:\r\n regex: filter.regex\r\n source_directory: source_directory\r\n run: '#filter_tool'\r\n out:\r\n - file_list\r\n flake8_step:\r\n in:\r\n file_list: filter_flake8_step/file_list\r\n source_directory: source_directory\r\n verbose: flake8.verbose\r\n run: '#flake8_tool'\r\n out:\r\n - flake8_report\r\n save_flake8_step:\r\n in:\r\n name: name\r\n pipeline_id: pipeline_id\r\n report: flake8_step/flake8_report\r\n run_id: run_id\r\n server_url: server_url\r\n run: '#save_tool'\r\n out: []\r\n id: flake8_subworkflow\r\n{% for tool in tools %}{{ tool.definition }}\r\n{% endfor %}", + "user_params": { + "filter": { + "regex": { + "type": "string", + "label": "regex", + "default": ".*\\.py" + } + }, + "flake8": { + "verbose": { + "doc": "Increase the verbosity of Flake8’s output.", + "type": "boolean", + "label": "Verbose", + "default": false + } + } + }, + "version": "0.1", + "tags": [ + 1, + 5 + ], + "tools": [ + "filter", + "flake8", + "save" + ] + } + }, + { + "model": "backend.subworkflow", + "pk": "notebook-bp-validator_subworkflow", + "fields": { + "name": "Jupyter Notebook Best Practices Validator", + "description": "This tool aims at validating the notebooks against the CEOS Jupyter Notebook Best Practice v1.1 document.", + "pipeline_step": "notebook-bp-validator_subworkflow:\r\n in:\r\n filter.regex: notebook-bp-validator_subworkflow.filter.regex\r\n pipeline_id: pipeline_id\r\n notebook-bp-validator.abspath: notebook-bp-validator_subworkflow.notebook-bp-validator.abspath\r\n notebook-bp-validator.schema: notebook-bp-validator_subworkflow.notebook-bp-validator.schema\r\n run_id: run_id\r\n server_url: server_url\r\n source_directory: clone_step/repo_directory\r\n run: '#notebook-bp-validator_subworkflow'\r\n out: []", + "definition": "- class: Workflow\r\n\r\n inputs:\r\n name:\r\n type: string\r\n default: notebook-bp-validator\r\n filter.regex: string\r\n pipeline_id: string\r\n notebook-bp-validator.abspath: boolean\r\n notebook-bp-validator.schema: string\r\n run_id: string\r\n server_url: string\r\n source_directory: Directory\r\n\r\n outputs:\r\n notebook-bp-validator_report:\r\n type: File\r\n outputSource: notebook-bp-validator_step/notebook-bp-validator_report\r\n\r\n steps:\r\n filter_notebook-bp-validator_step:\r\n in:\r\n regex: filter.regex\r\n source_directory: source_directory\r\n run: '#filter_tool'\r\n out:\r\n - file_list\r\n notebook-bp-validator_step:\r\n in:\r\n abspath: notebook-bp-validator.abspath \r\n file_list: filter_notebook-bp-validator_step/file_list\r\n source_directory: source_directory\r\n schema: notebook-bp-validator.schema\r\n run: '#notebook-bp-validator_tool'\r\n out:\r\n - notebook-bp-validator_report\r\n save_notebook-bp-validator_step:\r\n in:\r\n name: name\r\n pipeline_id: pipeline_id\r\n report: notebook-bp-validator_step/notebook-bp-validator_report\r\n run_id: run_id\r\n server_url: server_url\r\n run: '#save_tool'\r\n out: []\r\n id: notebook-bp-validator_subworkflow\r\n{% for tool in tools %}{{ tool.definition }}\r\n{% endfor %}", + "user_params": { + "filter": { + "regex": { + "type": "string", + "label": "regex", + "default": ".*\\.ipynb" + } + }, + "notebook-bp-validator": { + "schema": { + "doc": "Supported values: 'eumetsat' or 'schema.org'", + "type": "string", + "label": "Schema", + "default": "eumetsat" + }, + "abspath": { + "doc": "Uses absolute paths in output.", + "type": "boolean", + "label": "Absolute path", + "default": false + } + } + }, + "version": "0.1", + "tags": [ + 4, + 5 + ], + "tools": [ + "filter", + "notebook-bp-validator", + "save" + ] + } + }, + { + "model": "backend.subworkflow", + "pk": "papermill_subworkflow", + "fields": { + "name": "Papermill", + "description": "Papermill is a tool for parameterizing and executing Jupyter Notebooks.", + "pipeline_step": "papermill_subworkflow:\r\n in:\r\n filter.regex: papermill_subworkflow.filter.regex\r\n papermill.extract_requirements: papermill_subworkflow.papermill.extract_requirements\r\n papermill.extra_requirements: papermill_subworkflow.papermill.extra_requirements\r\n papermill.execution_parameters: papermill_subworkflow.papermill.execution_parameters\r\n pipeline_id: pipeline_id\r\n run_id: run_id\r\n server_url: server_url\r\n source_directory: clone_step/repo_directory\r\n run: '#papermill_subworkflow'\r\n out: []", + "definition": "- class: Workflow\r\n\r\n # --- This workflow runs Papermill on a single notebook and saves the result.\r\n\r\n id: papermill_execution_subworkflow\r\n\r\n inputs:\r\n notebook_path: string # Single notebook file\r\n\r\n # Inputs for papermill_tool\r\n source_directory: Directory\r\n extract_requirements: boolean?\r\n extra_requirements: string?\r\n execution_parameters: string?\r\n\r\n # Inputs for save_tool\r\n name: string\r\n pipeline_id: string\r\n run_id: string\r\n server_url: string\r\n\r\n outputs: []\r\n\r\n steps:\r\n papermill_step:\r\n in:\r\n notebook_path: notebook_path\r\n extract_requirements: extract_requirements\r\n extra_requirements: extra_requirements\r\n execution_parameters: execution_parameters\r\n source_directory: source_directory\r\n run: '#papermill_tool'\r\n out:\r\n - output_nb\r\n\r\n save_papermill_step:\r\n in:\r\n name: name\r\n instance: notebook_path\r\n pipeline_id: pipeline_id\r\n report: papermill_step/output_nb\r\n run_id: run_id\r\n server_url: server_url\r\n run: '#save_tool'\r\n out: []\r\n\r\n- class: Workflow\r\n\r\n # --- This workflow filters the input files then executes the above sub-workflow on each of them (scatter)\r\n\r\n id: papermill_subworkflow\r\n\r\n requirements:\r\n ScatterFeatureRequirement: {}\r\n\r\n inputs:\r\n filter.regex: string\r\n papermill.extract_requirements: boolean\r\n papermill.extra_requirements: string\r\n papermill.execution_parameters: string\r\n name:\r\n type: string\r\n default: papermill\r\n pipeline_id: string\r\n run_id: string\r\n server_url: string\r\n source_directory: Directory\r\n\r\n outputs: []\r\n\r\n steps:\r\n filter_papermill_step:\r\n in:\r\n regex: filter.regex\r\n source_directory: source_directory\r\n run: '#filter_tool'\r\n out:\r\n - file_list # Array of strings (the input for scatter)\r\n\r\n execute_notebooks_step:\r\n # Run the subworkflow defined above\r\n run: '#papermill_execution_subworkflow'\r\n # Scatter the above sub-workflow [Papermill => Save] over the array of files\r\n scatter: notebook_path\r\n in:\r\n # The list of notebooks filtered in the previous step\r\n notebook_path: filter_papermill_step/file_list\r\n # Other sub-workflow inputs\r\n source_directory: source_directory\r\n extract_requirements: papermill.extract_requirements\r\n extra_requirements: papermill.extra_requirements\r\n execution_parameters: papermill.execution_parameters\r\n name: name\r\n pipeline_id: pipeline_id\r\n run_id: run_id\r\n server_url: server_url\r\n\r\n out: []\r\n\r\n{% for tool in tools %}{{ tool.definition }}\r\n{% endfor %}", + "user_params": { + "filter": { + "regex": { + "type": "string", + "label": "regex", + "default": ".*\\.ipynb" + } + }, + "papermill": { + "extra_requirements": { + "doc": "Python requirements", + "type": "string", + "label": "Python requirements", + "default": "" + }, + "execution_parameters": { + "doc": "E.g. alpha=0.6, ratio=0.1", + "type": "string", + "label": "Execution parameters", + "default": "" + }, + "extract_requirements": { + "doc": "Extract requirements", + "type": "boolean", + "label": "Extract requirements", + "default": false + } + } + }, + "version": "0.1", + "tags": [ + 4, + 7 + ], + "tools": [ + "filter", + "papermill", + "save" + ] + } + }, + { + "model": "backend.subworkflow", + "pk": "pylint_subworkflow", + "fields": { + "name": "Pylint", + "description": "pylint - Static code analyser tool for Python", + "pipeline_step": "pylint_subworkflow:\r\n in:\r\n filter.regex: pylint_subworkflow.filter.regex\r\n pipeline_id: pipeline_id\r\n pylint.disable: pylint_subworkflow.pylint.disable\r\n pylint.errors_only: pylint_subworkflow.pylint.errors_only\r\n pylint.verbose: pylint_subworkflow.pylint.verbose\r\n run_id: run_id\r\n server_url: server_url\r\n source_directory: clone_step/repo_directory\r\n run: '#pylint_subworkflow'\r\n out: []", + "definition": "- class: Workflow\r\n\r\n inputs:\r\n name:\r\n type: string\r\n default: pylint\r\n filter.regex: string\r\n pipeline_id: string\r\n pylint.disable: string\r\n pylint.errors_only: boolean\r\n pylint.verbose: boolean\r\n run_id: string\r\n server_url: string\r\n source_directory: Directory\r\n\r\n outputs:\r\n pylint_report:\r\n type: File\r\n outputSource: pylint_step/pylint_report\r\n\r\n steps:\r\n filter_pylint_step:\r\n in:\r\n regex: filter.regex\r\n source_directory: source_directory\r\n run: '#filter_tool'\r\n out:\r\n - file_list\r\n pylint_step:\r\n in:\r\n disable: pylint.disable\r\n errors_only: pylint.errors_only\r\n file_list: filter_pylint_step/file_list\r\n source_directory: source_directory\r\n verbose: pylint.verbose\r\n run: '#pylint_tool'\r\n out:\r\n - pylint_report\r\n save_pylint_step:\r\n in:\r\n name: name\r\n pipeline_id: pipeline_id\r\n report: pylint_step/pylint_report\r\n run_id: run_id\r\n server_url: server_url\r\n run: '#save_tool'\r\n out: []\r\n id: pylint_subworkflow\r\n{% for tool in tools %}{{ tool.definition }}\r\n{% endfor %}", + "user_params": { + "filter": { + "regex": { + "type": "string", + "label": "regex", + "default": ".*\\.py" + } + }, + "pylint": { + "disable": { + "doc": "Disable the message, report, category or checker with the given id(s).", + "type": "string", + "label": "Disable IDs", + "default": "E0401" + }, + "verbose": { + "doc": "In verbose mode, extra non-checker-related info will be displayed.", + "type": "boolean", + "label": "Verbose", + "default": false + }, + "errors_only": { + "doc": "In error mode, messages with a category besides ERROR or FATAL are suppressed, and no reports are done by default. Error mode is compatible with disabling specific errors.", + "type": "boolean", + "label": "Errors only", + "default": false + } + } + }, + "version": "0.1", + "tags": [ + 1, + 5 + ], + "tools": [ + "filter", + "pylint", + "save" + ] + } + }, + { + "model": "backend.subworkflow", + "pk": "ruff_ipynb_subworkflow", + "fields": { + "name": "Ruff - Notebook", + "description": "Ruff - An extremely fast Python linter and code formatter, written in Rust", + "pipeline_step": "ruff_ipynb_subworkflow:\r\n in:\r\n filter.regex: ruff_ipynb_subworkflow.filter.regex\r\n pipeline_id: pipeline_id\r\n ruff.verbose: ruff_ipynb_subworkflow.ruff.verbose\r\n run_id: run_id\r\n server_url: server_url\r\n source_directory: clone_step/repo_directory\r\n run: '#ruff_ipynb_subworkflow'\r\n out: []", + "definition": "- class: Workflow\r\n\r\n inputs:\r\n name:\r\n type: string\r\n default: ruff\r\n filter.regex: string\r\n pipeline_id: string\r\n ruff.verbose: boolean\r\n run_id: string\r\n server_url: string\r\n source_directory: Directory\r\n\r\n outputs:\r\n ruff_report:\r\n type: File\r\n outputSource: ruff_step/ruff_report\r\n\r\n steps:\r\n filter_ruff_step:\r\n in:\r\n regex: filter.regex\r\n source_directory: source_directory\r\n run: '#filter_tool'\r\n out:\r\n - file_list\r\n ruff_step:\r\n in:\r\n file_list: filter_ruff_step/file_list\r\n source_directory: source_directory\r\n verbose: ruff.verbose\r\n run: '#ruff_tool'\r\n out:\r\n - ruff_report\r\n save_ruff_step:\r\n in:\r\n name: name\r\n pipeline_id: pipeline_id\r\n report: ruff_step/ruff_report\r\n run_id: run_id\r\n server_url: server_url\r\n run: '#save_tool'\r\n out: []\r\n id: ruff_ipynb_subworkflow\r\n{% for tool in tools %}{{ tool.definition }}\r\n{% endfor %}", + "user_params": { + "ruff": { + "verbose": { + "doc": "Enable verbose logging.", + "type": "boolean", + "label": "Verbose", + "default": false + } + }, + "filter": { + "regex": { + "type": "string", + "label": "regex", + "default": ".*\\.ipynb" + } + } + }, + "version": "0.1", + "tags": [ + 4, + 5 + ], + "tools": [ + "filter", + "ruff", + "save" + ] + } + }, + { + "model": "backend.subworkflow", + "pk": "ruff_subworkflow", + "fields": { + "name": "Ruff", + "description": "Ruff - An extremely fast Python linter and code formatter, written in Rust", + "pipeline_step": "ruff_subworkflow:\r\n in:\r\n filter.regex: ruff_subworkflow.filter.regex\r\n pipeline_id: pipeline_id\r\n ruff.verbose: ruff_subworkflow.ruff.verbose\r\n run_id: run_id\r\n server_url: server_url\r\n source_directory: clone_step/repo_directory\r\n run: '#ruff_subworkflow'\r\n out: []", + "definition": "- class: Workflow\r\n\r\n inputs:\r\n name:\r\n type: string\r\n default: ruff\r\n filter.regex: string\r\n pipeline_id: string\r\n ruff.verbose: boolean\r\n run_id: string\r\n server_url: string\r\n source_directory: Directory\r\n\r\n outputs:\r\n ruff_report:\r\n type: File\r\n outputSource: ruff_step/ruff_report\r\n\r\n steps:\r\n filter_ruff_step:\r\n in:\r\n regex: filter.regex\r\n source_directory: source_directory\r\n run: '#filter_tool'\r\n out:\r\n - file_list\r\n ruff_step:\r\n in:\r\n file_list: filter_ruff_step/file_list\r\n source_directory: source_directory\r\n verbose: ruff.verbose\r\n run: '#ruff_tool'\r\n out:\r\n - ruff_report\r\n save_ruff_step:\r\n in:\r\n name: name\r\n pipeline_id: pipeline_id\r\n report: ruff_step/ruff_report\r\n run_id: run_id\r\n server_url: server_url\r\n run: '#save_tool'\r\n out: []\r\n id: ruff_subworkflow\r\n{% for tool in tools %}{{ tool.definition }}\r\n{% endfor %}", + "user_params": { + "ruff": { + "verbose": { + "doc": "Enable verbose logging.", + "type": "boolean", + "label": "Verbose", + "default": false + } + }, + "filter": { + "regex": { + "type": "string", + "label": "regex", + "default": ".*\\.py" + } + } + }, + "version": "0.1", + "tags": [ + 1, + 5 + ], + "tools": [ + "filter", + "ruff", + "save" + ] + } + }, + { + "model": "backend.subworkflow", + "pk": "sonarqube", + "fields": { + "name": "SonarQube", + "description": "SonarQube - Code Quality, Security & Static Analysis Tool\r\n\r\nThis tool creates a project in our internal SonarQube server, sends it the code for analysis, and then retrieves the analysis results for storage in the database.", + "pipeline_step": "sonarqube_workflow_step:\r\n in:\r\n pipeline_id: pipeline_id\r\n repo_path: clone_step/repo_directory\r\n run_id: run_id\r\n server_url: server_url\r\n sonarqube_project_key: sonarqube_project_key\r\n sonarqube_project_name: sonarqube_project_name\r\n sonarqube_server: sonarqube_server\r\n sonarqube_token: sonarqube_token\r\n run: '#sonarqube_workflow'\r\n out: []", + "definition": "- class: Workflow\r\n\r\n inputs:\r\n name:\r\n type: string\r\n default: sonarqube\r\n pipeline_id:\r\n type: string\r\n repo_path:\r\n type: Directory\r\n run_id:\r\n type: string\r\n server_url:\r\n type: string\r\n sonarqube_project_key:\r\n type: string\r\n sonarqube_project_name:\r\n type: string\r\n sonarqube_server:\r\n type: string\r\n default: sonarqube-sonarqube.sonarqube:9000\r\n sonarqube_token:\r\n type: string\r\n\r\n outputs: []\r\n\r\n steps:\r\n save_sonarqube_step:\r\n in:\r\n name: name\r\n pipeline_id: pipeline_id\r\n report: sonarqube_get_report_step/sonarqube_report\r\n run_id: run_id\r\n server_url: server_url\r\n run: '#save_tool'\r\n out: []\r\n sonarqube_create_project_step:\r\n in:\r\n sonarqube_project_key: sonarqube_project_key\r\n sonarqube_project_name: sonarqube_project_name\r\n sonarqube_server: sonarqube_server\r\n sonarqube_token: sonarqube_token\r\n run: '#sonarqube_create_project_tool'\r\n out:\r\n - sonarqube_project_key\r\n - sonarqube_server\r\n - sonarqube_token\r\n sonarqube_get_report_step:\r\n in:\r\n sonarqube_project_key: sonarqube_scan_step/sonarqube_project_key\r\n sonarqube_server: sonarqube_scan_step/sonarqube_server\r\n sonarqube_token: sonarqube_scan_step/sonarqube_token\r\n run: '#sonarqube_get_report_tool'\r\n out:\r\n - sonarqube_project_key\r\n - sonarqube_server\r\n - sonarqube_token\r\n - sonarqube_report\r\n sonarqube_scan_step:\r\n in:\r\n sonarqube_project_key: sonarqube_create_project_step/sonarqube_project_key\r\n sonarqube_server: sonarqube_create_project_step/sonarqube_server\r\n sonarqube_token: sonarqube_create_project_step/sonarqube_token\r\n source_directory: repo_path\r\n run: '#sonarqube_scan_tool'\r\n out:\r\n - sonarqube_project_key\r\n - sonarqube_server\r\n - sonarqube_token\r\n id: sonarqube_workflow\r\n{% for tool in tools %}{{ tool.definition }}\r\n{% endfor %}", + "user_params": {}, + "version": "0.1", + "tags": [], + "tools": [ + "sonarqube_create_project", + "sonarqube_scan", + "sonarqube_get_report", + "save" + ] + } + }, + { + "model": "backend.subworkflow", + "pk": "trivy_subworkflow", + "fields": { + "name": "Trivy", + "description": "The all-in-one open source security scanner\r\nUse Trivy to find vulnerabilities (CVE) & misconfigurations (IaC) across code repositories, binary artifacts, container images, Kubernetes clusters, and more.", + "pipeline_step": "trivy_subworkflow:\r\n in:\r\n pipeline_id: pipeline_id\r\n run_id: run_id\r\n server_url: server_url\r\n trivy.image: trivy_subworkflow.trivy.image\r\n run: '#trivy_subworkflow'\r\n out: []", + "definition": "- class: Workflow\r\n\r\n inputs:\r\n name:\r\n type: string\r\n default: trivy\r\n pipeline_id: string\r\n run_id: string\r\n server_url: string\r\n trivy.image: string\r\n\r\n outputs: []\r\n\r\n steps:\r\n save_step:\r\n in:\r\n name: name\r\n pipeline_id: pipeline_id\r\n report: trivy_step/trivy_report\r\n run_id: run_id\r\n server_url: server_url\r\n run: '#save_tool'\r\n out: []\r\n trivy_step:\r\n in:\r\n image: trivy.image\r\n run: '#trivy_tool'\r\n out:\r\n - trivy_report\r\n id: trivy_subworkflow\r\n{% for tool in tools %}{{ tool.definition }}\r\n{% endfor %}", + "user_params": { + "trivy": { + "image": { + "type": "string", + "label": "Docker image", + "default": "alpine/git" + } + } + }, + "version": "0.1", + "tags": [ + 6, + 9 + ], + "tools": [ + "save", + "trivy" + ] + } + }, + { + "model": "backend.pipeline", + "pk": 6, + "fields": { + "name": "Python pipeline", + "description": "Runs a series of static analysis tools on python files", + "template": "#!/usr/bin/env cwltool\r\n\r\n$graph:\r\n- class: Workflow\r\n\r\n requirements:\r\n SubworkflowFeatureRequirement: {}\r\n\r\n inputs:\r\n{%- for tool in subworkflows %}\r\n {%- for tool_name, params in tool.user_params.items() %}\r\n {%- for param_name, param in params.items() %}\r\n {{ tool.slug }}.{{ tool_name }}.{{ param_name }}:\r\n label: \"{{ param.label or param_name }}\"\r\n {%- if param.doc %}\r\n doc: |-\r\n {{ param.doc }}\r\n {%- endif %}\r\n type: {{ param.type }}\r\n default: {{ param.default | tojson }}\r\n {%- endfor %}\r\n {%- endfor %}\r\n{%- endfor %}\r\n branch:\r\n type: string\r\n default: ''\r\n pipeline_id: string\r\n run_id: string\r\n server_url: string\r\n\r\n outputs: []\r\n\r\n steps:\r\n {% for tool in subworkflows %}{{ tool.pipeline_step | indent(4) }}\r\n {% endfor %}\r\n id: main\r\n{% for tool in subworkflows %}\r\n{{ tool.definition }}\r\n{% endfor %}\r\ncwlVersion: v1.2", + "default_inputs": {}, + "owner": null, + "created_at": null, + "edited_at": null, + "version": "0.1", + "tools": [ + "clone_subworkflow", + "bandit_subworkflow", + "flake8_subworkflow", + "pylint_subworkflow", + "ruff_subworkflow" + ] + } + }, + { + "model": "backend.pipeline", + "pk": 8, + "fields": { + "name": "Notebook pipeline", + "description": "Runs a static analysis and then executes Jupyter Notebooks files (ipynb).", + "template": "#!/usr/bin/env cwltool\r\n\r\n$graph:\r\n- class: Workflow\r\n\r\n requirements:\r\n SubworkflowFeatureRequirement: {}\r\n\r\n inputs:\r\n{%- for tool in subworkflows %}\r\n {%- for tool_name, params in tool.user_params.items() %}\r\n {%- for param_name, param in params.items() %}\r\n {{ tool.slug }}.{{ tool_name }}.{{ param_name }}:\r\n label: \"{{ param.label or param_name }}\"\r\n {%- if param.doc %}\r\n doc: |-\r\n {{ param.doc }}\r\n {%- endif %}\r\n type: {{ param.type }}\r\n default: {{ param.default | tojson }}\r\n {%- endfor %}\r\n {%- endfor %}\r\n{%- endfor %}\r\n branch:\r\n type: string\r\n default: ''\r\n pipeline_id: string\r\n run_id: string\r\n server_url: string\r\n\r\n outputs: []\r\n\r\n steps:\r\n {% for tool in subworkflows %}{{ tool.pipeline_step | indent(4) }}\r\n {% endfor %}\r\n id: main\r\n{% for tool in subworkflows %}\r\n{{ tool.definition }}\r\n{% endfor %}\r\ncwlVersion: v1.2", + "default_inputs": {}, + "owner": null, + "created_at": null, + "edited_at": null, + "version": "0.1", + "tools": [ + "clone_subworkflow", + "notebook-bp-validator_subworkflow", + "papermill_subworkflow", + "ruff_ipynb_subworkflow" + ] + } + }, + { + "model": "backend.pipeline", + "pk": 11, + "fields": { + "name": "Docker pipeline", + "description": "Finds vulnerabilities and misconfigurations in Docker images.", + "template": "#!/usr/bin/env cwltool\r\n\r\n$graph:\r\n- class: Workflow\r\n\r\n requirements:\r\n SubworkflowFeatureRequirement: {}\r\n\r\n inputs:\r\n{%- for tool in subworkflows %}\r\n {%- for tool_name, params in tool.user_params.items() %}\r\n {%- for param_name, param in params.items() %}\r\n {{ tool.slug }}.{{ tool_name }}.{{ param_name }}:\r\n label: \"{{ param.label or param_name }}\"\r\n {%- if param.doc %}\r\n doc: |-\r\n {{ param.doc }}\r\n {%- endif %}\r\n type: {{ param.type }}\r\n default: {{ param.default | tojson }}\r\n {%- endfor %}\r\n {%- endfor %}\r\n{%- endfor %}\r\n pipeline_id: string\r\n run_id: string\r\n server_url: string\r\n\r\n outputs: []\r\n\r\n steps:\r\n {% for tool in subworkflows %}{{ tool.pipeline_step | indent(4) }}\r\n {% endfor %}\r\n id: main\r\n{% for tool in subworkflows %}\r\n{{ tool.definition }}\r\n{% endfor %}\r\ncwlVersion: v1.2", + "default_inputs": {}, + "owner": null, + "created_at": null, + "edited_at": null, + "version": "0.1", + "tools": [ + "trivy_subworkflow" + ] + } + }, + { + "model": "backend.pipeline", + "pk": 17, + "fields": { + "name": "Notebook static analysis pipeline", + "description": "Runs static analysis on Jupyer Notebook files (ipynb)", + "template": "#!/usr/bin/env cwltool\r\n\r\n$graph:\r\n- class: Workflow\r\n\r\n requirements:\r\n SubworkflowFeatureRequirement: {}\r\n\r\n inputs:\r\n{%- for tool in subworkflows %}\r\n {%- for tool_name, params in tool.user_params.items() %}\r\n {%- for param_name, param in params.items() %}\r\n {{ tool.slug }}.{{ tool_name }}.{{ param_name }}:\r\n label: \"{{ param.label or param_name }}\"\r\n {%- if param.doc %}\r\n doc: |-\r\n {{ param.doc }}\r\n {%- endif %}\r\n type: {{ param.type }}\r\n default: {{ param.default | tojson }}\r\n {%- endfor %}\r\n {%- endfor %}\r\n{%- endfor %}\r\n pipeline_id: string\r\n run_id: string\r\n server_url: string\r\n\r\n outputs: []\r\n\r\n steps:\r\n {% for tool in subworkflows %}{{ tool.pipeline_step | indent(4) }}\r\n {% endfor %}\r\n id: main\r\n{% for tool in subworkflows %}\r\n{{ tool.definition }}\r\n{% endfor %}\r\ncwlVersion: v1.2", + "default_inputs": {}, + "owner": null, + "created_at": "2025-09-16T07:47:34.320Z", + "edited_at": "2025-11-18T10:55:05.144Z", + "version": "0.1", + "tools": [ + "clone_subworkflow", + "notebook-bp-validator_subworkflow" + ] + } + }, + { + "model": "backend.pipeline", + "pk": 18, + "fields": { + "name": "Ruff for python", + "description": "Ruff for python", + "template": "#!/usr/bin/env cwltool\n\n$graph:\n- class: Workflow\n\n requirements:\n SubworkflowFeatureRequirement: {}\n\n inputs:\n{%- for tool in subworkflows %}\n {%- for tool_name, params in tool.user_params.items() %}\n {%- for param_name, param in params.items() %}\n {{ tool.slug }}.{{ tool_name }}.{{ param_name }}:\n label: \"{{ param.label or param_name }}\"\n {%- if param.doc %}\n doc: |-\n {{ param.doc }}\n {%- endif %}\n type: {{ param.type }}\n default: {{ param.default | tojson }}\n {%- endfor %}\n {%- endfor %}\n{%- endfor %}\n pipeline_id: string\n run_id: string\n server_url: string\n\n outputs: []\n\n steps:\n {% for tool in subworkflows %}{{ tool.pipeline_step | indent(4) }}\n {% endfor %}\n id: main\n{% for tool in subworkflows %}\n{{ tool.definition }}\n{% endfor %}\ncwlVersion: v1.2\n", + "default_inputs": {}, + "owner": null, + "created_at": "2025-10-21T00:42:55.449Z", + "edited_at": "2025-10-21T00:42:55.449Z", + "version": "0.1", + "tools": [ + "clone_subworkflow", + "ruff_subworkflow" + ] + } + }, + { + "model": "backend.pipeline", + "pk": 19, + "fields": { + "name": "Notebook execution pipeline", + "description": "Executes selected notebooks using Papermill", + "template": "#!/usr/bin/env cwltool\r\n\r\n$graph:\r\n- class: Workflow\r\n\r\n requirements:\r\n SubworkflowFeatureRequirement: {}\r\n\r\n inputs:\r\n{%- for tool in subworkflows %}\r\n {%- for tool_name, params in tool.user_params.items() %}\r\n {%- for param_name, param in params.items() %}\r\n {{ tool.slug }}.{{ tool_name }}.{{ param_name }}:\r\n label: \"{{ param.label or param_name }}\"\r\n {%- if param.doc %}\r\n doc: |-\r\n {{ param.doc }}\r\n {%- endif %}\r\n type: {{ param.type }}\r\n default: {{ param.default | tojson }}\r\n {%- endfor %}\r\n {%- endfor %}\r\n{%- endfor %}\r\n pipeline_id: string\r\n run_id: string\r\n server_url: string\r\n\r\n outputs: []\r\n\r\n steps:\r\n {% for tool in subworkflows %}{{ tool.pipeline_step | indent(4) }}\r\n {% endfor %}\r\n id: main\r\n{% for tool in subworkflows %}\r\n{{ tool.definition }}\r\n{% endfor %}\r\ncwlVersion: v1.2", + "default_inputs": {}, + "owner": null, + "created_at": "2025-11-04T11:59:57.031Z", + "edited_at": "2025-11-25T12:05:05.664Z", + "version": "0.1", + "tools": [ + "papermill_subworkflow", + "clone_subworkflow" + ] + } + } +] \ No newline at end of file diff --git a/backend/backend/migrations/0005_jobreport_instance.py b/backend/backend/migrations/0005_jobreport_instance.py new file mode 100644 index 0000000..9c9c550 --- /dev/null +++ b/backend/backend/migrations/0005_jobreport_instance.py @@ -0,0 +1,18 @@ +# Generated by Django 5.1.8 on 2025-11-20 12:47 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('backend', '0004_create_views'), + ] + + operations = [ + migrations.AddField( + model_name='jobreport', + name='instance', + field=models.CharField(default='', max_length=200), + ), + ] \ No newline at end of file diff --git a/backend/backend/migrations/0006_tool_subworkflow_status_available.py b/backend/backend/migrations/0006_tool_subworkflow_status_available.py new file mode 100644 index 0000000..97dd227 --- /dev/null +++ b/backend/backend/migrations/0006_tool_subworkflow_status_available.py @@ -0,0 +1,23 @@ +# Generated by Django 5.1.8 on 2025-12-24 14:29 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('backend', '0005_jobreport_instance'), + ] + + operations = [ + migrations.AddField( + model_name='subworkflow', + name='available', + field=models.BooleanField(default=True), + ), + migrations.AddField( + model_name='subworkflow', + name='status', + field=models.CharField(choices=[('Beta', 'Beta'), ('Candidate', 'Candidate'), ('Disabled', 'Disabled'), ('Stable', 'Stable'), ('Deprecated', 'Deprecated')], default='Stable', max_length=20), + ), + ] diff --git a/backend/backend/models.py b/backend/backend/models.py index ca01678..af320fd 100644 --- a/backend/backend/models.py +++ b/backend/backend/models.py @@ -50,9 +50,10 @@ class JobReport(models.Model): name = models.SlugField(max_length=50) output = models.JSONField() created_at = models.DateTimeField(null=True) + instance = models.CharField(max_length=200, default="") def __str__(self): - return f"Run {self.run.id} ({self.run.pipeline.name}): {self.name} job" + return f"Run {self.run.id} ({self.run.pipeline.name}): {self.name} {'/' if self.instance else ''} {self.instance}" class Tag(models.Model): @@ -63,6 +64,14 @@ def __str__(self): class Subworkflow(models.Model): + + class Status(models.TextChoices): + BETA = 'Beta' + CANDIDATE = 'Candidate' + DISABLED = 'Disabled' + STABLE = 'Stable' + DEPRECATED = 'Deprecated' + slug = models.SlugField(primary_key=True, max_length=50) name = models.CharField(max_length=50) description = models.TextField(null=True) @@ -72,6 +81,8 @@ class Subworkflow(models.Model): tags = models.ManyToManyField(Tag, related_name="subworkflows", blank=True) tools = models.ManyToManyField("CommandLineTool", related_name="subworkflows") version = models.CharField(max_length=50) + status = models.CharField(max_length=20, choices=Status.choices, default=Status.STABLE) + available = models.BooleanField(default=True) def __str__(self): return self.name diff --git a/backend/backend/serializers.py b/backend/backend/serializers.py index 18890d8..40406ce 100644 --- a/backend/backend/serializers.py +++ b/backend/backend/serializers.py @@ -49,7 +49,7 @@ def get_job_reports_count(self, obj): class JobReportSerializer(serializers.ModelSerializer): class Meta: model = JobReport - fields = ["id", "name", "output", "run"] + fields = ["id", "name", "instance", "created_at", "output", "run"] read_only_fields = ["run"] @@ -64,6 +64,8 @@ class Meta: "tags", "tools", "version", + "status", + "available", ] diff --git a/backend/backend/urls.py b/backend/backend/urls.py index bbe260a..55386d3 100644 --- a/backend/backend/urls.py +++ b/backend/backend/urls.py @@ -6,7 +6,8 @@ JobReportViewSet, SubworkflowViewSet, TagViewSet, - SettingsView + SettingsView, + EventsView, ) router = DefaultRouter() @@ -21,4 +22,5 @@ urlpatterns += [ path("settings/", SettingsView.as_view(), name="settings"), + path("events/", EventsView.as_view(), name="events"), ] diff --git a/backend/backend/utils/cloudevents.py b/backend/backend/utils/cloudevents.py new file mode 100644 index 0000000..24e1089 --- /dev/null +++ b/backend/backend/utils/cloudevents.py @@ -0,0 +1,59 @@ +import json +import logging +import os +import time + +from cloudevents.conversion import to_structured +from cloudevents.http import CloudEvent + + +logger = logging.getLogger(__name__) + + +def decode(body, headers: dict) -> tuple[dict, dict]: + """ + Decode CloudEvent and return the payload and the headers + """ + headers_dict = {} + payload_dict = {} + try: + # Extract the event headers + for key, value in headers: + if key.startswith('HTTP_'): + # Convert 'HTTP_X_FORWARDED_FOR' to 'X-Forwarded-For' + header_name = key[5:].replace('_', '-').title() + headers_dict[header_name] = value + elif key in ('CONTENT_TYPE', 'CONTENT_LENGTH'): + # Add special headers not prefixed with HTTP_ + headers_dict[key.replace('_', '-').title()] = value + + # Extract the event data + if body: + payload_dict = json.loads(body.decode('utf-8')) + else: + payload_dict = {} + + logger.debug("Event Headers:\n%s", json.dumps(headers_dict, indent=2)) + logger.debug("Event Data:\n%s", json.dumps(payload_dict, indent=2)) + + except json.JSONDecodeError: + logger.error("JSONDecodeError") + return Response( + {"error": "Invalid JSON payload in event body"}, + status=status.HTTP_400_BAD_REQUEST + ) + + return payload_dict, headers_dict + + +def encode(attributes, data, headers=None): + event = CloudEvent(attributes, data) + logger.debug("Event: %s", event) + _ignore, payload = to_structured(event) + headers_dict = { + f"Ce-{k}": v + for k, v in attributes.items() + } + logger.debug("Headers: %s", headers_dict) + logger.debug("Payload: %s", payload) + return payload, headers_dict \ No newline at end of file diff --git a/backend/backend/utils/run_workflow.py b/backend/backend/utils/run_workflow.py index e8d498e..fbd6b7a 100644 --- a/backend/backend/utils/run_workflow.py +++ b/backend/backend/utils/run_workflow.py @@ -23,10 +23,10 @@ AQBB_SECRET = os.getenv("AQBB_SECRET", None) # Create a ServiceAccount for Calrissian with the right roles and use it here AQBB_SERVICEACCOUNT = os.getenv("AQBB_SERVICEACCOUNT", None) -# Backend service replicated in the vcluster (for reports storage) +# Backend service, possibly replicated in a virtual cluster (for storing reports) BACKEND_SERVICE_HOST = os.getenv( "BACKEND_SERVICE_HOST", - "backend-service.default.svc.cluster.local" + "application-quality-api.application-quality.svc.cluster.local" ) BACKEND_SERVICE_PORT = os.getenv("BACKEND_SERVICE_PORT", "80") SONARQUBE_SERVER = os.getenv( @@ -130,17 +130,6 @@ def run_workflow( # If cluster_config_file is None here, the ultimate option (if vclusters are not required) # is running the pipeline in the host cluster - namespace_name = f"applicationqualitypipeline-{run_id}" - session = CalrissianContext( - namespace=namespace_name, - kubeconfig_file=cluster_config_file, - storage_class=AQBB_STORAGECLASS, - volume_size=AQBB_VOLUMESIZE, - image_pull_secrets=AQBB_SECRET, - ) - - session.initialise() - # TODO: Remove Sonarqube parameters sonarqube_project = f"{username}-{pipeline_run.pipeline.pk}-{str(run_id)}" params = { @@ -163,6 +152,17 @@ def run_workflow( logger.debug("Run %s updated with server url", pipeline_run.id) logger.debug("Pipeline parameters: %s", params) + namespace_name = f"applicationqualitypipeline-{run_id}" + session = CalrissianContext( + namespace=namespace_name, + kubeconfig_file=cluster_config_file, + storage_class=AQBB_STORAGECLASS, + volume_size=AQBB_VOLUMESIZE, + image_pull_secrets=AQBB_SECRET, + ) + + session.initialise() + # # Create the Calrissian job # https://terradue.github.io/pycalrissian/gettingstarted/#create-the-calrissianjob diff --git a/backend/backend/views.py b/backend/backend/views.py index 04bb6ab..5c23725 100644 --- a/backend/backend/views.py +++ b/backend/backend/views.py @@ -1,9 +1,11 @@ import logging import os +import time import yaml from django.utils import timezone from django.db.utils import IntegrityError +from django.contrib.auth.models import User from jinja2 import Template from rest_framework import mixins, permissions, status, viewsets @@ -13,6 +15,7 @@ from backend.models import Pipeline, PipelineRun, JobReport, Subworkflow, Tag from backend.tasks import run_workflow_task +from backend.utils.cloudevents import encode, decode from . import serializers @@ -47,6 +50,95 @@ def get(self, request): return Response(settings) +RESPONSE_SOURCE = os.getenv("NOTIF_RESPONSE_SOURCE", "/eoepca/application-quality") +RESPONSE_TYPE_PREFIX = os.getenv( + "NOTIF_RESPONSE_TYPE_PREFIX", + "org.eoepca.application-quality.response" +) + + +class EventsView(APIView): + def post(self, request, *args, **kwargs): + try: + logger.info("Event received %s", request) + payload, headers = decode(request.body, request.META.items()) + event_id = headers.get('Ce-Id') + event_user = headers.get('Ce-User', None) + event_type = headers.get('Ce-Type', None) + event_subject = headers.get('Ce-Subject', None) + + user = None + pipeline_id = None + + if event_user: + user = User.objects.get(username=event_user) + logger.debug("User: %s", user) + else: + logger.debug("No user identified in the event") + + if event_subject and event_subject.startswith("pipelines/"): + pipeline_id = event_subject.split("/")[-1] + logger.debug("Pipeline: %s", pipeline_id) + else: + logger.debug("No pipeline identified in the event") + + # Default response data + res_data = { + "status": "unknown", + "processed_id": event_id, + "timestamp": int(time.time()), + "message": "Unknown event." + } + res_attrs = { + "id": event_id, + "source": RESPONSE_SOURCE, + "type": RESPONSE_TYPE_PREFIX + ".unknown", + "specversion": "1.0", + } + + # React to the event + # Note: The Trigger must filter on the event type prefix + + if event_type.endswith(".probes.health"): + logger.debug("Received a health check event") + res_data.update({ + "status": "ok", + # "processed_id": event_id, + "timestamp": int(time.time()), + "message": "Healthy.", + }) + res_attrs.update({ + "type": RESPONSE_TYPE_PREFIX + ".ok" + }) + + if user and pipeline_id and event_type.endswith(".event.pipeline.execute"): + pipeline_run = PipelineRunViewSet._create(user, pipeline_id, payload) + logger.debug("Pipeline run: %s", pipeline_run) + + res_data.update({ + "status": "accepted", + # "processed_id": event_id, + "timestamp": int(time.time()), + "message": "Pipeline execution created.", + }) + res_attrs.update({ + "type": RESPONSE_TYPE_PREFIX + ".accepted", + "subject": pipeline_run.id, + }) + + res_body, res_headers = encode(res_attrs, res_data) + logger.debug("Response Headers: %s", res_headers) + logger.debug("Response Body: %s", res_body) + return Response(res_body, status=202, headers=res_headers) + + except Exception as e: + logger.error("Exception: %s", e) + return Response( + {"error": str(e)}, + status=status.HTTP_500_INTERNAL_SERVER_ERROR + ) + + class IsOwnerOrAdmin(permissions.BasePermission): def has_object_permission(self, request, view, obj): if request.user.is_staff: @@ -100,10 +192,9 @@ def get_queryset(self): return PipelineRun.objects.filter(started_by=self.request.user) return PipelineRun.objects.filter(pipeline_id=p_id, started_by=self.request.user) - def create(self, request, *args, **kwargs): - user = self.request.user + @staticmethod + def _create(user: User, pipeline_id: str, data: dict) -> PipelineRun: logger.info("User %s is creating a pipeline run (admin=%s)", user, user.is_staff) - pipeline_id = self.kwargs["pipeline_id"] logger.info("Creating a new run for pipeline %s", pipeline_id) try: @@ -120,20 +211,19 @@ def create(self, request, *args, **kwargs): usage_report="", start_time=timezone.now(), status="starting", - started_by=request.user, + started_by=user, ) logger.info("Pipeline run created with id %s", pipeline_run.id) - yaml_cwl = self._render_cwl(pipeline) + yaml_cwl = PipelineRunViewSet._render_cwl(pipeline) cwl = yaml.safe_load(yaml_cwl) logger.info("Running workflow with id %s", pipeline_run.id) - payload = request.data # dict run_workflow_task.delay( run_id=pipeline_run.id, - parameters=payload.get("parameters"), + parameters=data.get("parameters"), cwl=cwl, - username=request.user.username, + username=user.username, ) pipeline_run.executed_cwl = yaml_cwl @@ -144,13 +234,25 @@ def create(self, request, *args, **kwargs): pipeline_run.save() logger.debug("Run %s updated with CWL and inputs", pipeline_run.id) + return pipeline_run + + def create(self, request, *args, **kwargs) -> Response: + user = self.request.user + # logger.info("User %s is creating a pipeline run (admin=%s)", user, user.is_staff) + pipeline_id = self.kwargs["pipeline_id"] + # logger.info("Creating a new run for pipeline %s", pipeline_id) + pipeline_run = PipelineRunViewSet._create(user, pipeline_id, request.data) + if isinstance(pipeline_run, Response): + # An error occurred which is described in the Response instance + return pipeline_run serializer = self.get_serializer(pipeline_run) return Response( serializer.data, status=status.HTTP_201_CREATED ) - def _merge_params(self, subworkflow: Subworkflow, default_inputs: dict) -> dict: + @staticmethod + def _merge_params(subworkflow: Subworkflow, default_inputs: dict) -> dict: if subworkflow.slug not in default_inputs: return subworkflow.user_params @@ -166,7 +268,8 @@ def _merge_params(self, subworkflow: Subworkflow, default_inputs: dict) -> dict: return merged_params - def _render_cwl(self, pipeline): + @staticmethod + def _render_cwl(pipeline): logger.debug("Rendering CWL for pipeline '%s'", pipeline.id) rendered_subworkflows = [] @@ -177,7 +280,7 @@ def _render_cwl(self, pipeline): subtool = { "definition": subtemplate.render(subcontext), "slug": subworkflow.pk, - "user_params": self._merge_params(subworkflow, pipeline.default_inputs), + "user_params": PipelineRunViewSet._merge_params(subworkflow, pipeline.default_inputs), "pipeline_step": subworkflow.pipeline_step, } rendered_subworkflows.append(subtool) @@ -205,6 +308,9 @@ def get_queryset(self): tool_name = self.request.query_params.get("name") if tool_name: queryset = queryset.filter(name=tool_name) + instance = self.request.query_params.get("instance") + if instance: + queryset = queryset.filter(instance=instance) return queryset @@ -216,6 +322,9 @@ def create(self, request, *args, **kwargs): tool_name = request.query_params.get("name") if not tool_name: raise ValidationError("Tool 'name' is required as a query parameter.") + + # The (optional) instance parameter allows to distinguish reports from scattered steps + instance = request.query_params.get("instance", "") try: run = PipelineRun.objects.get(pipeline__id=pipeline_id, id=run_id) @@ -230,25 +339,27 @@ def create(self, request, *args, **kwargs): status=status.HTTP_404_NOT_FOUND ) - if JobReport.objects.filter(run=run, name=tool_name).exists(): + if JobReport.objects.filter(run=run, name=tool_name, instance=instance).exists(): logger.warning( - "Couln't create a job report: A job report for '%s' already exists in run %s", + "Could not create a job report: A job report for '%s'/'%s' already exists in run %s", tool_name, + instance, run_id ) return Response( - {"error": f"A job report for '{tool_name}' already exists for this run."}, + {"error": f"A job report for '{tool_name}/{instance}' already exists for this run."}, status=status.HTTP_400_BAD_REQUEST, ) job_report = JobReport.objects.create( name=tool_name, + instance=instance, run=run, output=request.data, created_at=timezone.now() ) - logger.info("Job report created for tool '%s' in run %s", tool_name, run_id) + logger.info("Job report created for tool '%s'/'%s' in run %s", tool_name, instance, run_id) serializer = self.get_serializer(job_report) return Response( @@ -258,9 +369,20 @@ def create(self, request, *args, **kwargs): class SubworkflowViewSet(viewsets.ReadOnlyModelViewSet): - queryset = Subworkflow.objects.all() serializer_class = serializers.SubworkflowSerializer + def get_queryset(self): + user = self.request.user + if user: + logger.info(f"User {user} is requesting the tools information") + else: + logger.info(f"Anonymous user is requesting the tools information") + if user and user.is_staff: + logger.info(f"User {user} is staff / admin") + # Admins may get all the tools, whatever their status or availability flag + return Subworkflow.objects.all() + return Subworkflow.objects.filter(available=True) + class TagViewSet(viewsets.ReadOnlyModelViewSet): queryset = Tag.objects.all() diff --git a/backend/requirements.txt b/backend/requirements.txt index 76e001e..7ed2219 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -1,14 +1,16 @@ -celery[Redis] -cwltool +celery[Redis]==5.5.3 +cwltool==3.1.20251031082601 Django==5.1.8 -djangorestframework +djangorestframework==3.16.1 django-cors-headers==4.6.0 -djangorestframework-simplejwt +djangorestframework-simplejwt==5.5.1 +django-svelte-jsoneditor==0.4.4 # To serve admin static files in production whitenoise==6.8.2 # gunicorn -Jinja2 +Jinja2==3.1.6 kubernetes==28.1.0 -loguru -mozilla-django-oidc -psycopg2 \ No newline at end of file +loguru==0.7.3 +mozilla-django-oidc==4.0.1 +psycopg2==2.9.11 +cloudevents==1.12 \ No newline at end of file