From 5846f2241195cc4281a16d727b7b98211b01f2da Mon Sep 17 00:00:00 2001 From: Jia Wei Date: Thu, 19 Mar 2026 14:15:02 +0100 Subject: [PATCH 1/6] Added Gitleaks secret scanner to test workflow --- .github/workflows/tests.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index eb00360..871e732 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -6,7 +6,6 @@ on: jobs: test: runs-on: ubuntu-latest - steps: - name: Checkout code uses: actions/checkout@v4 @@ -25,6 +24,11 @@ jobs: echo "Generated services:" ls -la services/ || echo "No services directory" + - name: Run GitLeaks secret scanner + uses: gitleaks/gitleaks-action@v2 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: Install dependencies run: | python -m pip install --upgrade pip From e3c52a29b6a40692ec474a7971a7a3dd3696ea93 Mon Sep 17 00:00:00 2001 From: Jia Wei Date: Thu, 19 Mar 2026 14:38:33 +0100 Subject: [PATCH 2/6] Added YAML validity checks for test workflow --- .github/workflows/main.yml | 6 ++++++ .github/workflows/tests.yml | 15 +++++++++++++++ 2 files changed, 21 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 7f2dac4..0eef032 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -29,6 +29,12 @@ jobs: token: ${{ secrets.PAT_TOKEN }} path: notebooks-repo + - name: YAML Lint (Validity Only) + uses: ibiqlik/action-yamllint@v3 + with: + config: "{extends: relaxed, rules: {line-length: disable}}" + file_or_dir: . + # Filter the notebooks and no other files to create images - name: Filter changed notebooks run: | diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 871e732..375dae0 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -24,6 +24,21 @@ jobs: echo "Generated services:" ls -la services/ || echo "No services directory" + - name: YAML Validity check + uses: ibiqlik/action-yamllint@v3 + with: + file_or_dir: . + # This tells it to only care about syntax errors + config_data: | + extends: relaxed + rules: + line-length: disable + trailing-spaces: disable + new-line-at-end-of-file: disable + document-start: disable + indentation: disable + truthy: disable + - name: Run GitLeaks secret scanner uses: gitleaks/gitleaks-action@v2 env: From 0c1f1a1a0379bf8ab21d41cbab2046bd477b3a0c Mon Sep 17 00:00:00 2001 From: Jia Wei Date: Thu, 19 Mar 2026 14:55:30 +0100 Subject: [PATCH 3/6] Added new stage to run commit checks --- .github/workflows/tests.yml | 42 +++++++++++++++++++++++-------------- 1 file changed, 26 insertions(+), 16 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 375dae0..015f361 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -4,31 +4,16 @@ on: workflow_dispatch: jobs: - test: + commit-checks: runs-on: ubuntu-latest steps: - name: Checkout code uses: actions/checkout@v4 - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: "3.12" - - - name: Run generator (create Dockerfile, main.py and requirements.txt) - run: | - pip install nbformat - rm -rf src/services/ - cd src/ - python3 generator.py ../notebooks/ - echo "Generated services:" - ls -la services/ || echo "No services directory" - - name: YAML Validity check uses: ibiqlik/action-yamllint@v3 with: file_or_dir: . - # This tells it to only care about syntax errors config_data: | extends: relaxed rules: @@ -44,6 +29,31 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: Run Ruff + uses: astral-sh/ruff-action@v3 + with: + args: "check" + + test: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Run generator (create Dockerfile, main.py and requirements.txt) + run: | + pip install nbformat + rm -rf src/services/ + cd src/ + python3 generator.py ../notebooks/ + echo "Generated services:" + ls -la services/ || echo "No services directory" + - name: Install dependencies run: | python -m pip install --upgrade pip From 2c345459c37c786340a02bd9c890e2f3c8b0dcda Mon Sep 17 00:00:00 2001 From: Jia Wei Date: Fri, 20 Mar 2026 12:45:53 +0100 Subject: [PATCH 4/6] Configured Ruff in test workflow --- .github/workflows/tests.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 015f361..3f4c716 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -32,6 +32,7 @@ jobs: - name: Run Ruff uses: astral-sh/ruff-action@v3 with: + src: "./src" args: "check" test: From b283efac2b34607500edc3a9bfbfb4a28542d920 Mon Sep 17 00:00:00 2001 From: Jia Wei Date: Fri, 20 Mar 2026 13:50:38 +0100 Subject: [PATCH 5/6] Formatted files with Ruff --- .github/workflows/tests.yml | 2 +- src/generated_api_template.py | 61 +++++++++++------------- src/generated_helper.py | 88 ++++++++++++++++++++--------------- src/generator.py | 49 ++++++++++--------- 4 files changed, 102 insertions(+), 98 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 3f4c716..03c7a36 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -33,7 +33,7 @@ jobs: uses: astral-sh/ruff-action@v3 with: src: "./src" - args: "check" + args: "format --check" test: runs-on: ubuntu-latest diff --git a/src/generated_api_template.py b/src/generated_api_template.py index 16221e1..fd7f5df 100644 --- a/src/generated_api_template.py +++ b/src/generated_api_template.py @@ -11,7 +11,7 @@ import logging import sys -#Setup logging to console +# Setup logging to console logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) console_handler = logging.StreamHandler(sys.stdout) @@ -38,7 +38,8 @@ logger.info(f"Route path set as: {route_path}") parameterHandler = NotebookParameters(notebook_path) -PostModel = create_model("post_model",**parameterHandler.dynamic_model) +PostModel = create_model("post_model", **parameterHandler.dynamic_model) + def parse_output(output): """ @@ -47,13 +48,14 @@ def parse_output(output): match output.get("output_type"): case "stream": return handle_streams(output) - + case "execute_result" | "display_data": return handle_rich_outputs(output) - + case "error": return handle_errors(output) + def execute_notebook(notebook, params=None): logger.info(f"Executing notebook: {notebook}") @@ -62,67 +64,56 @@ def execute_notebook(notebook, params=None): with tempfile.NamedTemporaryFile(suffix=".ipynb") as tmp: try: if params is None: - pm.execute_notebook( - notebook, - tmp.name, - kernel_name="global_venv" - ) + pm.execute_notebook(notebook, tmp.name, kernel_name="global_venv") else: - pm.execute_notebook( - notebook, - tmp.name, - params, - kernel_name="global_venv" - ) - except PapermillExecutionError as e: + pm.execute_notebook(notebook, tmp.name, params, kernel_name="global_venv") + except PapermillExecutionError as e: pass # Read outputs nb = nbformat.read(tmp.name, as_version=4) for cell in nb.cells: - if cell.cell_type != "code": - continue + if cell.cell_type != "code": + continue - for output in cell.get("outputs", []): - results.append(parse_output(output)) + for output in cell.get("outputs", []): + results.append(parse_output(output)) return results except ValueError as e: return f"Error= {e}" + ############## # API Routes # ############## API_PREFIX = os.environ.get("API_PREFIX", "") -app = FastAPI(docs_url=f"{API_PREFIX}/docs",openapi_url=f"{API_PREFIX}/openapi.json") +app = FastAPI(docs_url=f"{API_PREFIX}/docs", openapi_url=f"{API_PREFIX}/openapi.json") + @app.get(f"{API_PREFIX}/") async def root(): return RedirectResponse(url=f"{API_PREFIX}{route_path}") + @app.get(f"{API_PREFIX}/getParameters") async def getParameters(): return parameterHandler.readable_json -@app.get(f"{API_PREFIX}{route_path}",response_model=NotebookResponse) + +@app.get(f"{API_PREFIX}{route_path}", response_model=NotebookResponse) async def endpoint(): - output = await asyncio.to_thread( - execute_notebook, - notebook_path, - params=None - ) + output = await asyncio.to_thread(execute_notebook, notebook_path, params=None) return NotebookResponse(outputs=output) -@app.post(f"{API_PREFIX}{route_path}",response_model=NotebookResponse) -async def executeWithParams(params:PostModel): - output = await asyncio.to_thread( - execute_notebook, - notebook_path, - params= params.model_dump() - ) + +@app.post(f"{API_PREFIX}{route_path}", response_model=NotebookResponse) +async def executeWithParams(params: PostModel): + output = await asyncio.to_thread(execute_notebook, notebook_path, params=params.model_dump()) return NotebookResponse(outputs=output) + @app.get(f"/health") async def health(): - return {"status": "ok"} \ No newline at end of file + return {"status": "ok"} diff --git a/src/generated_helper.py b/src/generated_helper.py index b61c3ac..80baf43 100644 --- a/src/generated_helper.py +++ b/src/generated_helper.py @@ -1,9 +1,11 @@ -from io import StringIO -from typing import Literal, Any, Annotated, Union -from pydantic import BaseModel, Field import ast -import nbformat import re +from io import StringIO +from typing import Annotated, Any, Literal, Union + +import nbformat +from pydantic import BaseModel, Field + # Pydantic API response model class StreamOutput(BaseModel): @@ -11,50 +13,61 @@ class StreamOutput(BaseModel): stream: Literal["stdout", "stderr"] content: str + class ImageOutput(BaseModel): type: Literal["image"] mime: Literal["image/png", "image/jpeg", "image/gif"] content: str # base64 encoded + class SvgOutput(BaseModel): type: Literal["svg"] content: str # raw XML string + class JsonOutput(BaseModel): type: Literal["json"] content: Any + class HtmlOutput(BaseModel): type: Literal["html"] content: str + class MarkdownOutput(BaseModel): type: Literal["markdown"] content: str + class LatexOutput(BaseModel): type: Literal["latex"] content: str + class TextOutput(BaseModel): type: Literal["text"] content: str + class ErrorOutput(BaseModel): type: Literal["error"] ename: str evalue: str traceback: list[str] + class DataframeOutput(BaseModel): type: Literal["dataframe"] content: str json: Any + class UnknownOutput(BaseModel): type: Literal["unknown"] data: dict[str, Any] + CellOutput = Annotated[ Union[ StreamOutput, @@ -67,22 +80,24 @@ class UnknownOutput(BaseModel): TextOutput, ErrorOutput, UnknownOutput, - DataframeOutput + DataframeOutput, ], - Field(discriminator="type") + Field(discriminator="type"), ] + class NotebookResponse(BaseModel): outputs: list[CellOutput] + class NotebookParameters: - def __init__(self,notebook_path): + def __init__(self, notebook_path): self.notebook_path = notebook_path self._parameter_cell = self._extract_parameters() self._parameter_nodes = self._infer_parameter_types() self.readable_json = self._to_readable_json() self.dynamic_model = self._to_dynamic_model() - + def _extract_parameters(self): """ Extracts cells tagged as Parameters from the notebook @@ -102,14 +117,14 @@ def _infer_parameter_types(self): if self._parameter_cell is None: return {} tree = ast.parse(self._parameter_cell) - parameterNodes:list = [] + parameterNodes: list = [] for node in tree.body: # Checks for variable assignment if isinstance(node, ast.Assign): parameterNodes.append(node) return parameterNodes - + def _to_readable_json(self): """ Format parameter type to readable json @@ -125,15 +140,12 @@ def _to_readable_json(self): # Evaluate to get value try: value = ast.literal_eval(node.value) - parameters[param_name] = { - "defaultValue": value, - "type": type(value).__name__ - } + parameters[param_name] = {"defaultValue": value, "type": type(value).__name__} except ValueError: # For complex types or expressions literal_eval can't handle parameters[param_name] = {"value": "Complex Expression", "type": "unknown"} return parameters - + def _to_dynamic_model(self): """ Format parameters types to Pydantic model @@ -148,26 +160,28 @@ def _to_dynamic_model(self): # Evaluate to get value try: value = ast.literal_eval(node.value) - parameters[param_name] = (type(value),value) + parameters[param_name] = (type(value), value) except ValueError: # For complex types or expressions literal_eval can't handle - parameters[param_name] = (type(object),"") + parameters[param_name] = (type(object), "") return parameters - + + def handle_streams(output): return { "type": "stream", "stream": output.get("name", "stdout"), - "content": "".join(output.text) if isinstance(output.text, list) else output.text + "content": "".join(output.text) if isinstance(output.text, list) else output.text, } + def handle_html(content): """ Parse HTML or Dataframe """ if " str: """ Normalizes file names into lower case and replaces spaces, special characters @@ -46,19 +53,15 @@ def extract_imports(notebook_path): imports.add(top_level) return sorted(imports) -def main(base_dir = None): - GENERATED_DIR = "services" - BASE_DIR = (sys.argv[1] if len(sys.argv) > 1 else os.getcwd()) if base_dir is None else base_dir - API_TEMPLATE_PATH = f"{pathlib.Path(__file__).resolve().parent}/generated_api_template.py" - DOCKER_TEMPLATE_PATH = f"{pathlib.Path(__file__).resolve().parent}/generated_dockerfile" - API_RESPONSE_TYPE_PATH = f"{pathlib.Path(__file__).resolve().parent}/generated_helper.py" - EXCLUDED_DIRS = {"venv", ".venv", ".git", "__pycache__", ".ipynb_checkpoints","services", "devops"} - + +def main(base_dir=None): + base_dir = (sys.argv[1] if len(sys.argv) > 1 else os.getcwd()) if base_dir is None else base_dir + if not os.path.exists(GENERATED_DIR): os.makedirs(GENERATED_DIR) detectedNotebooks = [] - for root, dirs, files in os.walk(BASE_DIR): + for root, dirs, files in os.walk(base_dir): dirs[:] = [d for d in dirs if d not in EXCLUDED_DIRS] for file in files: if file.endswith(".ipynb"): @@ -82,21 +85,16 @@ def main(base_dir = None): os.makedirs(service_folder) logger.info(f"Copying {full_path} to {service_folder}") - #Copy files to generated directory + # Copy files to generated directory shutil.copy(full_path, service_folder) - shutil.copy(API_TEMPLATE_PATH,f"{service_folder}/main.py") - shutil.copy(DOCKER_TEMPLATE_PATH,f"{service_folder}/dockerfile") - shutil.copy(API_RESPONSE_TYPE_PATH,f"{service_folder}/helper.py") + shutil.copy(API_TEMPLATE_PATH, f"{service_folder}/main.py") + shutil.copy(DOCKER_TEMPLATE_PATH, f"{service_folder}/dockerfile") + shutil.copy(API_RESPONSE_TYPE_PATH, f"{service_folder}/helper.py") # Generate Requirements.txt notebook_imports = extract_imports(full_path) - requirements = "\n".join([ - "fastapi", - "papermill", - "nbformat", - "uvicorn" - ] + notebook_imports) + requirements = "\n".join(["fastapi", "papermill", "nbformat", "uvicorn"] + notebook_imports) # Generate empty requirements.txt with open(os.path.join(service_folder, "requirements.txt"), "w") as f: @@ -104,5 +102,6 @@ def main(base_dir = None): logger.info(f"Generated service for notebook: {file}\n") + if __name__ == "__main__": - main() \ No newline at end of file + main() From ff99ab5ff14fcfa2181db733c52071587d364325 Mon Sep 17 00:00:00 2001 From: Jia Wei Date: Fri, 20 Mar 2026 14:01:06 +0100 Subject: [PATCH 6/6] Integrated commit checks to main workflow --- .github/workflows/main.yml | 37 +++++++++++++++++++++++++++++++------ 1 file changed, 31 insertions(+), 6 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 0eef032..3579dad 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -3,6 +3,37 @@ on: repository_dispatch: types: [notebooks-updated] jobs: + commit-checks: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: YAML Validity check + uses: ibiqlik/action-yamllint@v3 + with: + file_or_dir: . + config_data: | + extends: relaxed + rules: + line-length: disable + trailing-spaces: disable + new-line-at-end-of-file: disable + document-start: disable + indentation: disable + truthy: disable + + - name: Run GitLeaks secret scanner + uses: gitleaks/gitleaks-action@v2 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Run Ruff + uses: astral-sh/ruff-action@v3 + with: + src: "./src" + args: "format --check" + build: name: Build Docker Image runs-on: ubuntu-latest @@ -29,12 +60,6 @@ jobs: token: ${{ secrets.PAT_TOKEN }} path: notebooks-repo - - name: YAML Lint (Validity Only) - uses: ibiqlik/action-yamllint@v3 - with: - config: "{extends: relaxed, rules: {line-length: disable}}" - file_or_dir: . - # Filter the notebooks and no other files to create images - name: Filter changed notebooks run: |