From 385bc01c6c5bc1764f17a7f8ad90918af760c82d Mon Sep 17 00:00:00 2001 From: "D.Leap" Date: Tue, 21 Apr 2026 22:56:50 +0530 Subject: [PATCH] added library namespace, removed pyproject.toml addressing changes in pipelines and components. Signed-off-by: D.Leap --- __init__.py | 10 +- components/__init__.py | 12 -- components/data_processing/__init__.py | 8 -- .../tests/test_component_local.py | 12 -- .../tests/test_component_unit.py | 69 --------- components/deployment/__init__.py | 8 -- components/deployment/component_valid/OWNERS | 2 - components/evaluation/__init__.py | 8 -- components/training/__init__.py | 8 -- library/__init__.py | 1 + library/components/__init__.py | 5 + .../components}/data_processing/README.md | 0 .../components/data_processing/__init__.py | 4 + .../components}/data_processing/sdg/OWNERS | 0 .../components}/data_processing/sdg/README.md | 0 .../data_processing/sdg/__init__.py | 0 .../data_processing/sdg/component.py | 5 +- .../data_processing/sdg/metadata.yaml | 0 .../data_processing/sdg/shared/run_local.py | 0 .../data_processing/sdg/tests/__init__.py | 0 .../sdg/tests/test_component_local.py | 0 .../sdg/tests/test_component_unit.py | 0 .../sdg/tests/test_data/llm_test_flow.yaml | 0 .../test_data/prompts/generate_question.yaml | 0 .../sdg/tests/test_data/sample_input.jsonl | 0 .../tests/test_data/transform_test_flow.yaml | 0 .../yoda_data_processor/OWNERS | 0 .../yoda_data_processor/README.md | 21 +-- .../yoda_data_processor/__init__.py | 0 .../yoda_data_processor/component.py | 16 ++- .../yoda_data_processor/metadata.yaml | 0 .../yoda_data_processor/tests/__init__.py | 0 .../yoda_data_processor/tests/support.py | 108 ++++++++++++++ .../tests/test_component_local.py | 40 ++++++ .../tests/test_component_unit.py | 66 +++++++++ library/components/deployment/__init__.py | 4 + .../test_data/component_valid/OWNERS | 4 + .../test_data}/component_valid/metadata.yaml | 0 library/components/evaluation/__init__.py | 4 + library/components/training/__init__.py | 4 + library/pipelines/__init__.py | 5 + .../pipelines}/data_processing/README.md | 0 library/pipelines/data_processing/__init__.py | 4 + .../pipelines}/data_processing/sdg/OWNERS | 0 .../pipelines}/data_processing/sdg/README.md | 0 .../data_processing/sdg/__init__.py | 0 .../data_processing/sdg/metadata.yaml | 0 .../data_processing/sdg/pipeline.py | 6 +- .../data_processing/sdg/tests/__init__.py | 1 + .../sdg/tests/test_pipeline.py | 20 +++ library/pipelines/deployment/__init__.py | 4 + library/pipelines/evaluation/__init__.py | 4 + library/pipelines/training/__init__.py | 4 + pipelines/__init__.py | 13 -- pipelines/data_processing/__init__.py | 8 -- .../sdg/tests/data/llm_test_flow.yaml | 37 ----- .../tests/data/prompts/generate_question.yaml | 10 -- pipelines/deployment/__init__.py | 8 -- pipelines/evaluation/__init__.py | 8 -- pipelines/training/__init__.py | 8 -- pyproject.toml | 134 ------------------ 61 files changed, 305 insertions(+), 388 deletions(-) delete mode 100644 components/__init__.py delete mode 100644 components/data_processing/__init__.py delete mode 100644 components/data_processing/yoda_data_processor/tests/test_component_local.py delete mode 100644 components/data_processing/yoda_data_processor/tests/test_component_unit.py delete mode 100644 components/deployment/__init__.py delete mode 100644 components/deployment/component_valid/OWNERS delete mode 100644 components/evaluation/__init__.py delete mode 100644 components/training/__init__.py create mode 100644 library/__init__.py create mode 100644 library/components/__init__.py rename {components => library/components}/data_processing/README.md (100%) create mode 100644 library/components/data_processing/__init__.py rename {components => library/components}/data_processing/sdg/OWNERS (100%) rename {components => library/components}/data_processing/sdg/README.md (100%) rename {components => library/components}/data_processing/sdg/__init__.py (100%) rename {components => library/components}/data_processing/sdg/component.py (99%) rename {components => library/components}/data_processing/sdg/metadata.yaml (100%) rename {components => library/components}/data_processing/sdg/shared/run_local.py (100%) rename {components => library/components}/data_processing/sdg/tests/__init__.py (100%) rename {components => library/components}/data_processing/sdg/tests/test_component_local.py (100%) rename {components => library/components}/data_processing/sdg/tests/test_component_unit.py (100%) rename {components => library/components}/data_processing/sdg/tests/test_data/llm_test_flow.yaml (100%) rename {components => library/components}/data_processing/sdg/tests/test_data/prompts/generate_question.yaml (100%) rename {components => library/components}/data_processing/sdg/tests/test_data/sample_input.jsonl (100%) rename {components => library/components}/data_processing/sdg/tests/test_data/transform_test_flow.yaml (100%) rename {components => library/components}/data_processing/yoda_data_processor/OWNERS (100%) rename {components => library/components}/data_processing/yoda_data_processor/README.md (70%) rename {components => library/components}/data_processing/yoda_data_processor/__init__.py (100%) rename {components => library/components}/data_processing/yoda_data_processor/component.py (86%) rename {components => library/components}/data_processing/yoda_data_processor/metadata.yaml (100%) rename {components => library/components}/data_processing/yoda_data_processor/tests/__init__.py (100%) create mode 100644 library/components/data_processing/yoda_data_processor/tests/support.py create mode 100644 library/components/data_processing/yoda_data_processor/tests/test_component_local.py create mode 100644 library/components/data_processing/yoda_data_processor/tests/test_component_unit.py create mode 100644 library/components/deployment/__init__.py create mode 100644 library/components/deployment/test_data/component_valid/OWNERS rename {components/deployment => library/components/deployment/test_data}/component_valid/metadata.yaml (100%) create mode 100644 library/components/evaluation/__init__.py create mode 100644 library/components/training/__init__.py create mode 100644 library/pipelines/__init__.py rename {pipelines => library/pipelines}/data_processing/README.md (100%) create mode 100644 library/pipelines/data_processing/__init__.py rename {pipelines => library/pipelines}/data_processing/sdg/OWNERS (100%) rename {pipelines => library/pipelines}/data_processing/sdg/README.md (100%) rename {pipelines => library/pipelines}/data_processing/sdg/__init__.py (100%) rename {pipelines => library/pipelines}/data_processing/sdg/metadata.yaml (100%) rename {pipelines => library/pipelines}/data_processing/sdg/pipeline.py (95%) create mode 100644 library/pipelines/data_processing/sdg/tests/__init__.py create mode 100644 library/pipelines/data_processing/sdg/tests/test_pipeline.py create mode 100644 library/pipelines/deployment/__init__.py create mode 100644 library/pipelines/evaluation/__init__.py create mode 100644 library/pipelines/training/__init__.py delete mode 100644 pipelines/__init__.py delete mode 100644 pipelines/data_processing/__init__.py delete mode 100644 pipelines/data_processing/sdg/tests/data/llm_test_flow.yaml delete mode 100644 pipelines/data_processing/sdg/tests/data/prompts/generate_question.yaml delete mode 100644 pipelines/deployment/__init__.py delete mode 100644 pipelines/evaluation/__init__.py delete mode 100644 pipelines/training/__init__.py delete mode 100644 pyproject.toml diff --git a/__init__.py b/__init__.py index b79403a77..6785b1224 100644 --- a/__init__.py +++ b/__init__.py @@ -3,9 +3,9 @@ A collection of reusable components and pipelines for Kubeflow Pipelines. Usage: - from kfp_components import components, pipelines - from kfp_components.components import training - from kfp_components.pipelines import evaluation + from library import components, pipelines + from library.components import training + from library.pipelines import evaluation """ # Import submodules to enable the convenient import patterns shown above @@ -17,5 +17,5 @@ # Fallback to absolute imports (works during testing with sys.path modification) import components # noqa: F401 import pipelines # noqa: F401 - -__all__ = ["components", "pipelines"] +#since the +__all__ = ["pipelines", "components"] diff --git a/components/__init__.py b/components/__init__.py deleted file mode 100644 index 3b904b548..000000000 --- a/components/__init__.py +++ /dev/null @@ -1,12 +0,0 @@ -"""Kubeflow Pipelines Components - Core Components Package - -This module auto-imports all components for clean usage: - from kfp_components.components import training - from kfp_components.components import evaluation - from kfp_components.components import data_processing - from kfp_components.components import deployment -""" - -from . import data_processing, deployment, evaluation, training - -__all__ = ["data_processing", "deployment", "evaluation", "training"] diff --git a/components/data_processing/__init__.py b/components/data_processing/__init__.py deleted file mode 100644 index 161238215..000000000 --- a/components/data_processing/__init__.py +++ /dev/null @@ -1,8 +0,0 @@ -"""Data Processing Components Module - -This module re-exports all components in the data_processing category for easy import: - from kfp_components.components.data_processing import component_name -""" - -# Auto-generated imports will be added here by scripts/update_init_imports.py -# Components will be imported dynamically based on subdirectories diff --git a/components/data_processing/yoda_data_processor/tests/test_component_local.py b/components/data_processing/yoda_data_processor/tests/test_component_local.py deleted file mode 100644 index 744f20d8b..000000000 --- a/components/data_processing/yoda_data_processor/tests/test_component_local.py +++ /dev/null @@ -1,12 +0,0 @@ -"""Tests for the yoda_data_processor component.""" - -from ..component import prepare_yoda_dataset - - -class TestYodaDataProcessorLocalRunner: - """Test component with LocalRunner (subprocess execution).""" - - def test_local_execution(self, setup_and_teardown_subprocess_runner): # noqa: F811 - """Test component execution with LocalRunner.""" - # Execute the component - prepare_yoda_dataset(yoda_input_dataset="dvgodoy/yoda_sentences") diff --git a/components/data_processing/yoda_data_processor/tests/test_component_unit.py b/components/data_processing/yoda_data_processor/tests/test_component_unit.py deleted file mode 100644 index 9ea502177..000000000 --- a/components/data_processing/yoda_data_processor/tests/test_component_unit.py +++ /dev/null @@ -1,69 +0,0 @@ -"""Tests for the yoda_data_processor component.""" - -from unittest import mock - -from ..component import prepare_yoda_dataset - - -class TestYodaDataProcessorUnitTests: - """Unit tests for component logic.""" - - def test_component_function_exists(self): - """Test that the component function is properly imported.""" - assert callable(prepare_yoda_dataset) - assert hasattr(prepare_yoda_dataset, "python_func") - - @mock.patch.dict("sys.modules", {"datasets": mock.MagicMock()}) - @mock.patch("datasets.load_dataset") - def test_component_with_default_parameters(self, mock_load_dataset): - """Test component with default train_split_ratio.""" - # Setup mock dataset with proper chaining - mock_dataset = mock.MagicMock() - - # Configure length to return 100 consistently - mock_dataset.__len__.return_value = 100 - - # Ensure all transformation methods return the same mock for chaining - mock_dataset.rename_column.return_value = mock_dataset - mock_dataset.remove_columns.return_value = mock_dataset - mock_dataset.map.return_value = mock_dataset - - # Setup train/test split return values - mock_train = mock.MagicMock() - mock_train.__len__.return_value = 80 - mock_train.save_to_disk = mock.MagicMock() - - mock_test = mock.MagicMock() - mock_test.__len__.return_value = 20 - mock_test.save_to_disk = mock.MagicMock() - - split_result = {"train": mock_train, "test": mock_test} - mock_dataset.train_test_split.return_value = split_result - - mock_load_dataset.return_value = mock_dataset - - # Mock output datasets - mock_train_output = mock.MagicMock() - mock_train_output.path = "/tmp/train" - mock_eval_output = mock.MagicMock() - mock_eval_output.path = "/tmp/eval" - - # Call the actual python function - prepare_yoda_dataset.python_func( - yoda_input_dataset="test-dataset", yoda_train_dataset=mock_train_output, yoda_eval_dataset=mock_eval_output - ) - - # Verify interactions - mock_load_dataset.assert_called_once_with("test-dataset", split="train") - # Component makes two rename_column calls - expected_rename_calls = [mock.call("sentence", "prompt"), mock.call("translation_extra", "completion")] - mock_dataset.rename_column.assert_has_calls(expected_rename_calls) - mock_dataset.remove_columns.assert_called_once_with(["translation"]) - mock_dataset.map.assert_called_once() - # Check train_test_split call (with floating point tolerance for test_size) - assert mock_dataset.train_test_split.call_count == 1 - call_args, call_kwargs = mock_dataset.train_test_split.call_args - assert call_kwargs["seed"] == 42 - assert abs(call_kwargs["test_size"] - 0.2) < 1e-10 # Allow for floating point precision - mock_train.save_to_disk.assert_called_once_with("/tmp/train") - mock_test.save_to_disk.assert_called_once_with("/tmp/eval") diff --git a/components/deployment/__init__.py b/components/deployment/__init__.py deleted file mode 100644 index 604e0e352..000000000 --- a/components/deployment/__init__.py +++ /dev/null @@ -1,8 +0,0 @@ -"""Deployment Components Module - -This module re-exports all components in the deployment category for easy import: - from kfp_components.components.deployment import component_name -""" - -# Auto-generated imports will be added here by scripts/update_init_imports.py -# Components will be imported dynamically based on subdirectories diff --git a/components/deployment/component_valid/OWNERS b/components/deployment/component_valid/OWNERS deleted file mode 100644 index fa71bd17a..000000000 --- a/components/deployment/component_valid/OWNERS +++ /dev/null @@ -1,2 +0,0 @@ -approvers: -- sample-approver diff --git a/components/evaluation/__init__.py b/components/evaluation/__init__.py deleted file mode 100644 index 1f97ab09c..000000000 --- a/components/evaluation/__init__.py +++ /dev/null @@ -1,8 +0,0 @@ -"""Evaluation Components Module - -This module re-exports all components in the evaluation category for easy import: - from kfp_components.components.evaluation import component_name -""" - -# Auto-generated imports will be added here by scripts/update_init_imports.py -# Components will be imported dynamically based on subdirectories diff --git a/components/training/__init__.py b/components/training/__init__.py deleted file mode 100644 index c78fc3720..000000000 --- a/components/training/__init__.py +++ /dev/null @@ -1,8 +0,0 @@ -"""Training Components Module - -This module re-exports all components in the training category for easy import: - from kfp_components.components.training import component_name -""" - -# Auto-generated imports will be added here by scripts/update_init_imports.py -# Components will be imported dynamically based on subdirectories diff --git a/library/__init__.py b/library/__init__.py new file mode 100644 index 000000000..d63764efb --- /dev/null +++ b/library/__init__.py @@ -0,0 +1 @@ +"""Toplevel package for reusable library components and pipelines.""" diff --git a/library/components/__init__.py b/library/components/__init__.py new file mode 100644 index 000000000..96ab02cf4 --- /dev/null +++ b/library/components/__init__.py @@ -0,0 +1,5 @@ +"""Convenience imports for the reusable component categories in ``library``.""" + +from . import data_processing, deployment, evaluation, training + +__all__ = ["data_processing", "deployment", "evaluation", "training"] diff --git a/components/data_processing/README.md b/library/components/data_processing/README.md similarity index 100% rename from components/data_processing/README.md rename to library/components/data_processing/README.md diff --git a/library/components/data_processing/__init__.py b/library/components/data_processing/__init__.py new file mode 100644 index 000000000..471198f08 --- /dev/null +++ b/library/components/data_processing/__init__.py @@ -0,0 +1,4 @@ +"""Data processing components exposed under ``library.components.data_processing``.""" + +# Auto-generated imports will be added here by scripts/update_init_imports.py +# Components will be imported dynamically based on subdirectories diff --git a/components/data_processing/sdg/OWNERS b/library/components/data_processing/sdg/OWNERS similarity index 100% rename from components/data_processing/sdg/OWNERS rename to library/components/data_processing/sdg/OWNERS diff --git a/components/data_processing/sdg/README.md b/library/components/data_processing/sdg/README.md similarity index 100% rename from components/data_processing/sdg/README.md rename to library/components/data_processing/sdg/README.md diff --git a/components/data_processing/sdg/__init__.py b/library/components/data_processing/sdg/__init__.py similarity index 100% rename from components/data_processing/sdg/__init__.py rename to library/components/data_processing/sdg/__init__.py diff --git a/components/data_processing/sdg/component.py b/library/components/data_processing/sdg/component.py similarity index 99% rename from components/data_processing/sdg/component.py rename to library/components/data_processing/sdg/component.py index 0bd480512..0cb4f4d03 100644 --- a/components/data_processing/sdg/component.py +++ b/library/components/data_processing/sdg/component.py @@ -4,7 +4,6 @@ within Kubeflow Pipelines. """ -import kfp.compiler from kfp import dsl @@ -262,7 +261,9 @@ def sdg( if __name__ == "__main__": - kfp.compiler.Compiler().compile( + from kfp import compiler + + compiler.Compiler().compile( sdg, package_path=__file__.replace(".py", "_component.yaml"), ) diff --git a/components/data_processing/sdg/metadata.yaml b/library/components/data_processing/sdg/metadata.yaml similarity index 100% rename from components/data_processing/sdg/metadata.yaml rename to library/components/data_processing/sdg/metadata.yaml diff --git a/components/data_processing/sdg/shared/run_local.py b/library/components/data_processing/sdg/shared/run_local.py similarity index 100% rename from components/data_processing/sdg/shared/run_local.py rename to library/components/data_processing/sdg/shared/run_local.py diff --git a/components/data_processing/sdg/tests/__init__.py b/library/components/data_processing/sdg/tests/__init__.py similarity index 100% rename from components/data_processing/sdg/tests/__init__.py rename to library/components/data_processing/sdg/tests/__init__.py diff --git a/components/data_processing/sdg/tests/test_component_local.py b/library/components/data_processing/sdg/tests/test_component_local.py similarity index 100% rename from components/data_processing/sdg/tests/test_component_local.py rename to library/components/data_processing/sdg/tests/test_component_local.py diff --git a/components/data_processing/sdg/tests/test_component_unit.py b/library/components/data_processing/sdg/tests/test_component_unit.py similarity index 100% rename from components/data_processing/sdg/tests/test_component_unit.py rename to library/components/data_processing/sdg/tests/test_component_unit.py diff --git a/components/data_processing/sdg/tests/test_data/llm_test_flow.yaml b/library/components/data_processing/sdg/tests/test_data/llm_test_flow.yaml similarity index 100% rename from components/data_processing/sdg/tests/test_data/llm_test_flow.yaml rename to library/components/data_processing/sdg/tests/test_data/llm_test_flow.yaml diff --git a/components/data_processing/sdg/tests/test_data/prompts/generate_question.yaml b/library/components/data_processing/sdg/tests/test_data/prompts/generate_question.yaml similarity index 100% rename from components/data_processing/sdg/tests/test_data/prompts/generate_question.yaml rename to library/components/data_processing/sdg/tests/test_data/prompts/generate_question.yaml diff --git a/components/data_processing/sdg/tests/test_data/sample_input.jsonl b/library/components/data_processing/sdg/tests/test_data/sample_input.jsonl similarity index 100% rename from components/data_processing/sdg/tests/test_data/sample_input.jsonl rename to library/components/data_processing/sdg/tests/test_data/sample_input.jsonl diff --git a/components/data_processing/sdg/tests/test_data/transform_test_flow.yaml b/library/components/data_processing/sdg/tests/test_data/transform_test_flow.yaml similarity index 100% rename from components/data_processing/sdg/tests/test_data/transform_test_flow.yaml rename to library/components/data_processing/sdg/tests/test_data/transform_test_flow.yaml diff --git a/components/data_processing/yoda_data_processor/OWNERS b/library/components/data_processing/yoda_data_processor/OWNERS similarity index 100% rename from components/data_processing/yoda_data_processor/OWNERS rename to library/components/data_processing/yoda_data_processor/OWNERS diff --git a/components/data_processing/yoda_data_processor/README.md b/library/components/data_processing/yoda_data_processor/README.md similarity index 70% rename from components/data_processing/yoda_data_processor/README.md rename to library/components/data_processing/yoda_data_processor/README.md index b8ee8d559..69f28c3f8 100644 --- a/components/data_processing/yoda_data_processor/README.md +++ b/library/components/data_processing/yoda_data_processor/README.md @@ -20,26 +20,7 @@ Downloads the yoda_sentences dataset from HuggingFace, renames columns to match ## Metadata 🗂️ -- **Name**: yoda_data_processor -- **Stability**: alpha -- **Dependencies**: - - Kubeflow: - - Name: Pipelines, Version: >=2.15.2 - - External Services: - - Name: HuggingFace Datasets, Version: >=4.4.2 -- **Tags**: - - data_processing - - dataset_preparation - - text_processing - - yoda_speak - - translation -- **Last Verified**: 2025-12-19 11:30:16+00:00 -- **Owners**: - - Approvers: - - mprahl - - nsingla - - Reviewers: - - HumairAK +See [metadata.yaml](metadata.yaml) for the component's tags, dependencies, owners, and last verification date. ## Additional Resources 📚 diff --git a/components/data_processing/yoda_data_processor/__init__.py b/library/components/data_processing/yoda_data_processor/__init__.py similarity index 100% rename from components/data_processing/yoda_data_processor/__init__.py rename to library/components/data_processing/yoda_data_processor/__init__.py diff --git a/components/data_processing/yoda_data_processor/component.py b/library/components/data_processing/yoda_data_processor/component.py similarity index 86% rename from components/data_processing/yoda_data_processor/component.py rename to library/components/data_processing/yoda_data_processor/component.py index cfb95b987..672c80c69 100644 --- a/components/data_processing/yoda_data_processor/component.py +++ b/library/components/data_processing/yoda_data_processor/component.py @@ -1,7 +1,13 @@ -import kfp.compiler from kfp import dsl +def add_yoda_prefix(example: dict[str, str]) -> dict[str, str]: + """Prefix a prompt with the Yoda translation instruction.""" + updated_example = dict(example) + updated_example["prompt"] = "Translate the following to Yoda speak: " + updated_example["prompt"] + return updated_example + + @dsl.component( packages_to_install=["datasets"], ) @@ -35,10 +41,6 @@ def prepare_yoda_dataset( # Add prefix to prompts print("Adding Yoda speak prefix to prompts") - def add_yoda_prefix(example): - example["prompt"] = "Translate the following to Yoda speak: " + example["prompt"] - return example - dataset = dataset.map(add_yoda_prefix) # Split the dataset into train and eval sets @@ -63,7 +65,9 @@ def add_yoda_prefix(example): if __name__ == "__main__": - kfp.compiler.Compiler().compile( + from kfp import compiler + + compiler.Compiler().compile( prepare_yoda_dataset, package_path=__file__.replace(".py", "_component.yaml"), ) diff --git a/components/data_processing/yoda_data_processor/metadata.yaml b/library/components/data_processing/yoda_data_processor/metadata.yaml similarity index 100% rename from components/data_processing/yoda_data_processor/metadata.yaml rename to library/components/data_processing/yoda_data_processor/metadata.yaml diff --git a/components/data_processing/yoda_data_processor/tests/__init__.py b/library/components/data_processing/yoda_data_processor/tests/__init__.py similarity index 100% rename from components/data_processing/yoda_data_processor/tests/__init__.py rename to library/components/data_processing/yoda_data_processor/tests/__init__.py diff --git a/library/components/data_processing/yoda_data_processor/tests/support.py b/library/components/data_processing/yoda_data_processor/tests/support.py new file mode 100644 index 000000000..e716f87ea --- /dev/null +++ b/library/components/data_processing/yoda_data_processor/tests/support.py @@ -0,0 +1,108 @@ +"""Shared test helpers for yoda_data_processor tests.""" + +import json +import random +from pathlib import Path + + +class InMemoryDataset: + """A tiny dataset double that preserves transformation behavior.""" + + def __init__(self, rows: list[dict[str, str]]): + """Initialize the dataset with a copy of the provided rows.""" + self.rows = [dict(row) for row in rows] + + def __len__(self) -> int: + """Return the number of rows in the dataset.""" + return len(self.rows) + + def rename_column(self, old_name: str, new_name: str) -> "InMemoryDataset": + """Return a dataset with one column renamed.""" + return InMemoryDataset( + [ + { + (new_name if key == old_name else key): value + for key, value in row.items() + } + for row in self.rows + ] + ) + + def remove_columns(self, columns: list[str]) -> "InMemoryDataset": + """Return a dataset without the requested columns.""" + columns_to_remove = set(columns) + return InMemoryDataset( + [ + { + key: value + for key, value in row.items() + if key not in columns_to_remove + } + for row in self.rows + ] + ) + + def map(self, transform) -> "InMemoryDataset": + """Return a dataset with the transform applied to each row.""" + return InMemoryDataset([transform(dict(row)) for row in self.rows]) + + def train_test_split(self, test_size: float, seed: int) -> dict[str, "InMemoryDataset"]: + """Split the dataset deterministically into train and test subsets.""" + shuffled_rows = [dict(row) for row in self.rows] + random.Random(seed).shuffle(shuffled_rows) + + test_count = min(len(shuffled_rows), max(1, int(round(len(shuffled_rows) * test_size)))) + split_index = len(shuffled_rows) - test_count + return { + "train": InMemoryDataset(shuffled_rows[:split_index]), + "test": InMemoryDataset(shuffled_rows[split_index:]), + } + + def save_to_disk(self, path: str) -> None: + """Persist rows in a simple JSONL file within the output directory.""" + output_dir = Path(path) + output_dir.mkdir(parents=True, exist_ok=True) + with output_dir.joinpath("data.jsonl").open("w", encoding="utf-8") as handle: + for row in self.rows: + handle.write(json.dumps(row) + "\n") + + +def sample_rows() -> list[dict[str, str]]: + """Return a representative input dataset for Yoda processor tests.""" + return [ + { + "sentence": "Train yourself to let go of everything you fear to lose.", + "translation_extra": "Let go of everything you fear to lose, train yourself to.", + "translation": "unused-1", + }, + { + "sentence": "Do or do not. There is no try.", + "translation_extra": "Do or do not. Try, there is not.", + "translation": "unused-2", + }, + { + "sentence": "Named must your fear be before banish it you can.", + "translation_extra": "Before banish it you can, named must your fear be.", + "translation": "unused-3", + }, + { + "sentence": "Wars not make one great.", + "translation_extra": "Great, wars make one not.", + "translation": "unused-4", + }, + { + "sentence": "Pass on what you have learned.", + "translation_extra": "What you have learned, pass on.", + "translation": "unused-5", + }, + ] + + +def read_saved_rows(path: Path) -> list[dict[str, str]]: + """Load the saved JSONL rows from a component output directory.""" + data_file = path / "data.jsonl" + return [ + json.loads(line) + for line in data_file.read_text(encoding="utf-8").splitlines() + if line + ] diff --git a/library/components/data_processing/yoda_data_processor/tests/test_component_local.py b/library/components/data_processing/yoda_data_processor/tests/test_component_local.py new file mode 100644 index 000000000..9d4528608 --- /dev/null +++ b/library/components/data_processing/yoda_data_processor/tests/test_component_local.py @@ -0,0 +1,40 @@ +"""Tests for the yoda_data_processor component.""" + +import types +from pathlib import Path +from types import SimpleNamespace +from unittest import mock + +from ..component import prepare_yoda_dataset +from .support import InMemoryDataset, read_saved_rows, sample_rows + + +class TestYodaDataProcessorLocalRunner: + """Test local component execution without a live network dependency.""" + + def test_local_execution_writes_expected_outputs(self, setup_and_teardown_subprocess_runner, tmp_path: Path): + """Exercise the component locally with a stub datasets module and assert on outputs.""" + mock_load_dataset = mock.Mock(return_value=InMemoryDataset(sample_rows())) + fake_datasets_module = types.ModuleType("datasets") + fake_datasets_module.load_dataset = mock_load_dataset + + train_output = SimpleNamespace(path=str(tmp_path / "train")) + eval_output = SimpleNamespace(path=str(tmp_path / "eval")) + + with mock.patch.dict("sys.modules", {"datasets": fake_datasets_module}): + prepare_yoda_dataset.python_func( + yoda_train_dataset=train_output, + yoda_eval_dataset=eval_output, + ) + + mock_load_dataset.assert_called_once_with("dvgodoy/yoda_sentences", split="train") + + train_rows = read_saved_rows(Path(train_output.path)) + eval_rows = read_saved_rows(Path(eval_output.path)) + + assert Path(train_output.path).exists() + assert Path(eval_output.path).exists() + assert len(train_rows) == 4 + assert len(eval_rows) == 1 + assert train_rows + eval_rows + assert all(row["prompt"].startswith("Translate the following to Yoda speak: ") for row in train_rows + eval_rows) diff --git a/library/components/data_processing/yoda_data_processor/tests/test_component_unit.py b/library/components/data_processing/yoda_data_processor/tests/test_component_unit.py new file mode 100644 index 000000000..b3901577a --- /dev/null +++ b/library/components/data_processing/yoda_data_processor/tests/test_component_unit.py @@ -0,0 +1,66 @@ +"""Tests for the yoda_data_processor component.""" + +import types +from pathlib import Path +from types import SimpleNamespace +from unittest import mock + +from ..component import add_yoda_prefix, prepare_yoda_dataset +from .support import InMemoryDataset, read_saved_rows, sample_rows + + +class TestYodaDataProcessorUnitTests: + """Unit tests for component logic.""" + + def test_add_yoda_prefix_updates_prompt_without_mutating_input(self): + """Test the prompt transformation directly with a representative row.""" + example = { + "prompt": "Train yourself to let go of everything you fear to lose.", + "completion": "Let go of everything you fear to lose, train yourself to.", + } + + transformed = add_yoda_prefix(example) + + assert transformed == { + "prompt": "Translate the following to Yoda speak: Train yourself to let go of everything you fear to lose.", + "completion": "Let go of everything you fear to lose, train yourself to.", + } + assert example["prompt"] == "Train yourself to let go of everything you fear to lose." + + def test_component_writes_prefixed_train_and_eval_datasets(self, tmp_path: Path): + """Test the component against a small in-memory dataset fixture.""" + mock_load_dataset = mock.Mock(return_value=InMemoryDataset(sample_rows())) + fake_datasets_module = types.ModuleType("datasets") + fake_datasets_module.load_dataset = mock_load_dataset + + train_output = SimpleNamespace(path=str(tmp_path / "train")) + eval_output = SimpleNamespace(path=str(tmp_path / "eval")) + + with mock.patch.dict("sys.modules", {"datasets": fake_datasets_module}): + prepare_yoda_dataset.python_func( + yoda_input_dataset="test-dataset", + yoda_train_dataset=train_output, + yoda_eval_dataset=eval_output, + train_split_ratio=0.6, + ) + + mock_load_dataset.assert_called_once_with("test-dataset", split="train") + + train_rows = read_saved_rows(Path(train_output.path)) + eval_rows = read_saved_rows(Path(eval_output.path)) + combined_rows = train_rows + eval_rows + + assert len(train_rows) == 3 + assert len(eval_rows) == 2 + assert all(set(row) == {"prompt", "completion"} for row in combined_rows) + assert all(row["prompt"].startswith("Translate the following to Yoda speak: ") for row in combined_rows) + assert { + (row["prompt"], row["completion"]) + for row in combined_rows + } == { + ( + f"Translate the following to Yoda speak: {row['sentence']}", + row["translation_extra"], + ) + for row in sample_rows() + } diff --git a/library/components/deployment/__init__.py b/library/components/deployment/__init__.py new file mode 100644 index 000000000..11206d5eb --- /dev/null +++ b/library/components/deployment/__init__.py @@ -0,0 +1,4 @@ +"""Deployment components exposed under ``library.components.deployment``.""" + +# Auto-generated imports will be added here by scripts/update_init_imports.py +# Components will be imported dynamically based on subdirectories diff --git a/library/components/deployment/test_data/component_valid/OWNERS b/library/components/deployment/test_data/component_valid/OWNERS new file mode 100644 index 000000000..0ae03ef85 --- /dev/null +++ b/library/components/deployment/test_data/component_valid/OWNERS @@ -0,0 +1,4 @@ +approvers: + - test-owner +reviewers: + - test-reviewer diff --git a/components/deployment/component_valid/metadata.yaml b/library/components/deployment/test_data/component_valid/metadata.yaml similarity index 100% rename from components/deployment/component_valid/metadata.yaml rename to library/components/deployment/test_data/component_valid/metadata.yaml diff --git a/library/components/evaluation/__init__.py b/library/components/evaluation/__init__.py new file mode 100644 index 000000000..6efdc2e04 --- /dev/null +++ b/library/components/evaluation/__init__.py @@ -0,0 +1,4 @@ +"""Evaluation components exposed under ``library.components.evaluation``.""" + +# Auto-generated imports will be added here by scripts/update_init_imports.py +# Components will be imported dynamically based on subdirectories diff --git a/library/components/training/__init__.py b/library/components/training/__init__.py new file mode 100644 index 000000000..96fe76f4a --- /dev/null +++ b/library/components/training/__init__.py @@ -0,0 +1,4 @@ +"""Training components exposed under ``library.components.training``.""" + +# Auto-generated imports will be added here by scripts/update_init_imports.py +# Components will be imported dynamically based on subdirectories diff --git a/library/pipelines/__init__.py b/library/pipelines/__init__.py new file mode 100644 index 000000000..fa2dc2e8a --- /dev/null +++ b/library/pipelines/__init__.py @@ -0,0 +1,5 @@ +"""Convenience imports for the reusable pipeline categories in ``library``.""" + +from . import data_processing, deployment, evaluation, training + +__all__ = ["data_processing", "deployment", "evaluation", "training"] diff --git a/pipelines/data_processing/README.md b/library/pipelines/data_processing/README.md similarity index 100% rename from pipelines/data_processing/README.md rename to library/pipelines/data_processing/README.md diff --git a/library/pipelines/data_processing/__init__.py b/library/pipelines/data_processing/__init__.py new file mode 100644 index 000000000..4488a1f2f --- /dev/null +++ b/library/pipelines/data_processing/__init__.py @@ -0,0 +1,4 @@ +"""Data processing pipelines exposed under ``library.pipelines.data_processing``.""" + +# Auto-generated imports will be added here by scripts/update_init_imports.py +# Pipelines will be imported dynamically based on subdirectories diff --git a/pipelines/data_processing/sdg/OWNERS b/library/pipelines/data_processing/sdg/OWNERS similarity index 100% rename from pipelines/data_processing/sdg/OWNERS rename to library/pipelines/data_processing/sdg/OWNERS diff --git a/pipelines/data_processing/sdg/README.md b/library/pipelines/data_processing/sdg/README.md similarity index 100% rename from pipelines/data_processing/sdg/README.md rename to library/pipelines/data_processing/sdg/README.md diff --git a/pipelines/data_processing/sdg/__init__.py b/library/pipelines/data_processing/sdg/__init__.py similarity index 100% rename from pipelines/data_processing/sdg/__init__.py rename to library/pipelines/data_processing/sdg/__init__.py diff --git a/pipelines/data_processing/sdg/metadata.yaml b/library/pipelines/data_processing/sdg/metadata.yaml similarity index 100% rename from pipelines/data_processing/sdg/metadata.yaml rename to library/pipelines/data_processing/sdg/metadata.yaml diff --git a/pipelines/data_processing/sdg/pipeline.py b/library/pipelines/data_processing/sdg/pipeline.py similarity index 95% rename from pipelines/data_processing/sdg/pipeline.py rename to library/pipelines/data_processing/sdg/pipeline.py index 1352633f7..9c26b3a34 100644 --- a/pipelines/data_processing/sdg/pipeline.py +++ b/library/pipelines/data_processing/sdg/pipeline.py @@ -4,9 +4,9 @@ and produces generated output as a KFP artifact. """ -from kfp import compiler, dsl +from kfp import dsl from kfp.kubernetes import use_config_map_as_volume, use_secret_as_env -from kfp_components.components.data_processing.sdg.component import sdg +from library.components.data_processing.sdg.component import sdg @dsl.component(packages_to_install=["pandas"]) @@ -80,6 +80,8 @@ def sdg_llm_pipeline( if __name__ == "__main__": + from kfp import compiler + compiler.Compiler().compile( sdg_llm_pipeline, package_path="sdg_llm_pipeline.yaml", diff --git a/library/pipelines/data_processing/sdg/tests/__init__.py b/library/pipelines/data_processing/sdg/tests/__init__.py new file mode 100644 index 000000000..0e547cc2f --- /dev/null +++ b/library/pipelines/data_processing/sdg/tests/__init__.py @@ -0,0 +1 @@ +"""Tests for the SDG pipeline package.""" diff --git a/library/pipelines/data_processing/sdg/tests/test_pipeline.py b/library/pipelines/data_processing/sdg/tests/test_pipeline.py new file mode 100644 index 000000000..0dda73276 --- /dev/null +++ b/library/pipelines/data_processing/sdg/tests/test_pipeline.py @@ -0,0 +1,20 @@ +"""Tests for the SDG pipeline.""" + +from pathlib import Path + +from kfp import compiler + +from ..pipeline import sdg_llm_pipeline + + +def test_pipeline_compiles(tmp_path: Path): + """Test that the pipeline compiles successfully from the library package.""" + output_path = tmp_path / "sdg_llm_pipeline.yaml" + + compiler.Compiler().compile( + pipeline_func=sdg_llm_pipeline, + package_path=str(output_path), + ) + + assert output_path.exists() + assert "sdg-llm-test-pipeline" in output_path.read_text(encoding="utf-8") diff --git a/library/pipelines/deployment/__init__.py b/library/pipelines/deployment/__init__.py new file mode 100644 index 000000000..317d02cf4 --- /dev/null +++ b/library/pipelines/deployment/__init__.py @@ -0,0 +1,4 @@ +"""Deployment pipelines exposed under ``library.pipelines.deployment``.""" + +# Auto-generated imports will be added here by scripts/update_init_imports.py +# Pipelines will be imported dynamically based on subdirectories diff --git a/library/pipelines/evaluation/__init__.py b/library/pipelines/evaluation/__init__.py new file mode 100644 index 000000000..ac02a633d --- /dev/null +++ b/library/pipelines/evaluation/__init__.py @@ -0,0 +1,4 @@ +"""Evaluation pipelines exposed under ``library.pipelines.evaluation``.""" + +# Auto-generated imports will be added here by scripts/update_init_imports.py +# Pipelines will be imported dynamically based on subdirectories diff --git a/library/pipelines/training/__init__.py b/library/pipelines/training/__init__.py new file mode 100644 index 000000000..7109937b6 --- /dev/null +++ b/library/pipelines/training/__init__.py @@ -0,0 +1,4 @@ +"""Training pipelines exposed under ``library.pipelines.training``.""" + +# Auto-generated imports will be added here by scripts/update_init_imports.py +# Pipelines will be imported dynamically based on subdirectories diff --git a/pipelines/__init__.py b/pipelines/__init__.py deleted file mode 100644 index abe1dad83..000000000 --- a/pipelines/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -"""Kubeflow Pipelines Components - Core Pipelines Package - -This module auto-imports all pipelines for clean usage: - from kfp_components import pipelines - from kfp_components.pipelines import training - from kfp_components.pipelines import evaluation - from kfp_components.pipelines import data_processing - from kfp_components.pipelines import deployment -""" - -from . import data_processing, deployment, evaluation, training - -__all__ = ["data_processing", "deployment", "evaluation", "training"] diff --git a/pipelines/data_processing/__init__.py b/pipelines/data_processing/__init__.py deleted file mode 100644 index 0ed09f964..000000000 --- a/pipelines/data_processing/__init__.py +++ /dev/null @@ -1,8 +0,0 @@ -"""Data Processing Pipelines Module - -This module re-exports all pipelines in the data_processing category for easy import: - from kfp_components.pipelines.data_processing import pipeline_name -""" - -# Auto-generated imports will be added here by scripts/update_init_imports.py -# Pipelines will be imported dynamically based on subdirectories diff --git a/pipelines/data_processing/sdg/tests/data/llm_test_flow.yaml b/pipelines/data_processing/sdg/tests/data/llm_test_flow.yaml deleted file mode 100644 index 9c95af04c..000000000 --- a/pipelines/data_processing/sdg/tests/data/llm_test_flow.yaml +++ /dev/null @@ -1,37 +0,0 @@ ---- -metadata: - name: LLM Test Flow - description: Minimal LLM flow for KFP component E2E testing. - version: 1.0.0 - author: SDG Hub Team - tags: - - testing - license: Apache-2.0 - dataset_requirements: - required_columns: - - document - - domain - description: Requires document and domain columns. - id: llm-test-flow -blocks: - - block_type: PromptBuilderBlock - block_config: - block_name: build_question_prompt - input_cols: - - document - output_cols: question_prompt - prompt_config_path: prompts/generate_question.yaml - - block_type: LLMChatBlock - block_config: - block_name: generate_question - input_cols: question_prompt - output_cols: raw_question - max_tokens: 256 - temperature: 0.7 - async_mode: false - - block_type: LLMResponseExtractorBlock - block_config: - block_name: extract_question - input_cols: raw_question - extract_content: true - expand_lists: true diff --git a/pipelines/data_processing/sdg/tests/data/prompts/generate_question.yaml b/pipelines/data_processing/sdg/tests/data/prompts/generate_question.yaml deleted file mode 100644 index 204a4b91b..000000000 --- a/pipelines/data_processing/sdg/tests/data/prompts/generate_question.yaml +++ /dev/null @@ -1,10 +0,0 @@ ---- -- role: system - content: You are a helpful assistant that generates a short question about the given text. - -- role: user - content: | - Generate one short question about this text: - {{document}} - - Respond with only the question, nothing else. diff --git a/pipelines/deployment/__init__.py b/pipelines/deployment/__init__.py deleted file mode 100644 index 2afc16916..000000000 --- a/pipelines/deployment/__init__.py +++ /dev/null @@ -1,8 +0,0 @@ -"""Deployment Pipelines Module - -This module re-exports all pipelines in the deployment category for easy import: - from kfp_components.pipelines.deployment import pipeline_name -""" - -# Auto-generated imports will be added here by scripts/update_init_imports.py -# Pipelines will be imported dynamically based on subdirectories diff --git a/pipelines/evaluation/__init__.py b/pipelines/evaluation/__init__.py deleted file mode 100644 index a79297fc7..000000000 --- a/pipelines/evaluation/__init__.py +++ /dev/null @@ -1,8 +0,0 @@ -"""Evaluation Pipelines Module - -This module re-exports all pipelines in the evaluation category for easy import: - from kfp_components.pipelines.evaluation import pipeline_name -""" - -# Auto-generated imports will be added here by scripts/update_init_imports.py -# Pipelines will be imported dynamically based on subdirectories diff --git a/pipelines/training/__init__.py b/pipelines/training/__init__.py deleted file mode 100644 index 5fe8780e6..000000000 --- a/pipelines/training/__init__.py +++ /dev/null @@ -1,8 +0,0 @@ -"""Training Pipelines Module - -This module re-exports all pipelines in the training category for easy import: - from kfp_components.pipelines.training import pipeline_name -""" - -# Auto-generated imports will be added here by scripts/update_init_imports.py -# Pipelines will be imported dynamically based on subdirectories diff --git a/pyproject.toml b/pyproject.toml deleted file mode 100644 index 6796650cc..000000000 --- a/pyproject.toml +++ /dev/null @@ -1,134 +0,0 @@ -[build-system] -requires = ["setuptools", "wheel"] -build-backend = "setuptools.build_meta" - -[project] -name = "kfp-components" -version = "1.11.0" -description = "A collection of reusable components and pipelines for Kubeflow Pipelines" -authors = [ - {name = "Kubeflow Community"} -] -license = "Apache-2.0" -readme = "README.md" -requires-python = ">=3.11" -dependencies = [ - "kfp>=2.15.2", -] - -[project.optional-dependencies] -lint = [ - # We keep these pinned to avoid new lint failures due to updates - "ruff==0.15.2", - "yamllint==1.38.0", -] -test = [ - "docker", - "pytest", - "pytest-cov", - "pytest-timeout", - "docstring-parser", - "jinja2", - "packaging", - "pyyaml", - "semver", - "pip", - "setuptools", - # Component-specific SDK deps required for python_func() tests. - # KFP SubprocessRunner cannot auto-install these because it does not - # support Input[Dataset] artifacts (see: https://github.com/kubeflow/pipelines/issues/13061). - # Once KFP fixes this, these can be removed and tests can use - # SubprocessRunner(use_venv=True) instead. - "sdg-hub>=0.7.0,<1.0", - # Required for pipeline validation (validate_examples, validate_base_images) - "kfp-kubernetes", -] -# Local development (extends test with additional tools) -dev = [ - "kfp-components[lint]", - "kfp-components[test]", -] - -[project.urls] -Homepage = "https://www.kubeflow.org/docs/components/pipelines/" -Documentation = "https://github.com/kubeflow/pipelines-components/tree/main/docs" -Repository = "https://github.com/kubeflow/pipelines-components" -Issues = "https://github.com/kubeflow/pipelines-components/issues" - -[tool.setuptools] -packages = [ - "kfp_components", - "kfp_components.components", - "kfp_components.components.data_processing", - "kfp_components.components.data_processing.yoda_data_processor", - "kfp_components.components.data_processing.sdg", - "kfp_components.components.deployment", - "kfp_components.components.evaluation", - "kfp_components.components.training", - "kfp_components.pipelines", - "kfp_components.pipelines.data_processing", - "kfp_components.pipelines.data_processing.sdg", - "kfp_components.pipelines.deployment", - "kfp_components.pipelines.evaluation", - "kfp_components.pipelines.training", -] - -[tool.setuptools.package-dir] -"kfp_components" = "." -"kfp_components.components" = "components" -"kfp_components.pipelines" = "pipelines" - -[tool.setuptools.package-data] -"*" = ["*.yaml", "*.yml", "*.json"] - -[tool.ruff] -line-length = 120 -target-version = "py311" -exclude = [ - ".git", - ".venv", - "build", - "dist", - "__pycache__", - "node_modules", - "**/_generated/", - "**/test_data/**", # Test fixtures may have intentional styles (quote types, etc.) -] - -[tool.ruff.lint] -# Enable pycodestyle (E, W), pyflakes (F), isort (I), pydocstyle (D) -select = ["E", "W", "F", "I", "D"] -ignore = [ - "D100", # Missing docstring in public module - "D104", # Missing docstring in public package - "D415", # First line should end with punctuation (not needed for simple module names) -] - - -[tool.ruff.lint.pydocstyle] -convention = "google" - -[tool.ruff.lint.isort] -known-first-party = ["kubeflow"] - -[tool.ruff.format] -quote-style = "double" -indent-style = "space" -skip-magic-trailing-comma = false -docstring-code-format = true - -[project.scripts] -# Format is: script-name = "module.path:function_name" -my-script = "scripts.generate_readme.cli:main" - - -# Pytest configuration for KEP-913 repository structure -# Currently tests are in scripts/ only. When component/pipeline tests are added, -# this configuration will need to be updated to handle tests in components/*/tests/ -[tool.pytest.ini_options] -testpaths = ["scripts"] -python_files = ["test_*.py"] -python_classes = ["Test*"] -python_functions = ["test_*"] -# Don't search these directories -norecursedirs = [".git", ".venv", "build", "dist", "__pycache__", "components", "pipelines"]