diff --git a/converters/dbt/tests/__snapshots__/test_msi_to_ossie.ambr b/converters/dbt/tests/__snapshots__/test_msi_to_ossie.ambr index 88d1f9b8..a795f9ae 100644 --- a/converters/dbt/tests/__snapshots__/test_msi_to_ossie.ambr +++ b/converters/dbt/tests/__snapshots__/test_msi_to_ossie.ambr @@ -17,6 +17,7 @@ # under the License. # name: TestMetricConversion.test_derived_metric_nested ''' + version: 0.2.0.dev0 name: semantic_model datasets: - name: orders @@ -63,12 +64,12 @@ dialects: - dialect: ANSI_SQL expression: (SUM(orders.amount) - SUM(orders.cost_amount)) - SUM(orders.expense_amount) - version: 0.2.0.dev0 ''' # --- # name: TestMetricConversion.test_ratio_metric_inlines_sub_expressions ''' + version: 0.2.0.dev0 name: semantic_model datasets: - name: orders @@ -101,12 +102,12 @@ - dialect: ANSI_SQL expression: (SUM(orders.amount)) / (SUM(CASE WHEN orders.order_id IS NOT NULL THEN 1 ELSE 0 END)) - version: 0.2.0.dev0 ''' # --- # name: TestMetricFilterFlattening.test_metric_and_measure_filters_combined_with_and ''' + version: 0.2.0.dev0 name: semantic_model datasets: - name: orders @@ -124,12 +125,12 @@ - dialect: ANSI_SQL expression: SUM(CASE WHEN (status = 'paid') AND (region = 'intl') THEN orders.amount END) - version: 0.2.0.dev0 ''' # --- # name: TestRelationshipConversion.test_three_datasets_produce_all_pairs ''' + version: 0.2.0.dev0 name: semantic_model datasets: - name: users_a @@ -182,7 +183,6 @@ - user_id to_columns: - user_id - version: 0.2.0.dev0 ''' # --- diff --git a/converters/dbt/tests/__snapshots__/test_ossie_to_msi.ambr b/converters/dbt/tests/__snapshots__/test_ossie_to_msi.ambr index 7b89427a..103c3889 100644 --- a/converters/dbt/tests/__snapshots__/test_ossie_to_msi.ambr +++ b/converters/dbt/tests/__snapshots__/test_ossie_to_msi.ambr @@ -17,6 +17,7 @@ # under the License. # name: TestOssieToMSIRoundTrip.test_ossie_to_msi_to_ossie_preserves_structure ''' + version: 0.2.0.dev0 name: semantic_model datasets: - name: orders @@ -56,7 +57,6 @@ dialects: - dialect: ANSI_SQL expression: SUM(orders.amount) - version: 0.2.0.dev0 ''' # --- diff --git a/converters/dbt/tests/test_msi_to_ossie.py b/converters/dbt/tests/test_msi_to_ossie.py index 7ed0aa89..e74e671a 100644 --- a/converters/dbt/tests/test_msi_to_ossie.py +++ b/converters/dbt/tests/test_msi_to_ossie.py @@ -1209,6 +1209,7 @@ def test_to_ossie_json_produces_valid_json(self) -> None: parsed = json.loads(result.to_ossie_json()) assert parsed["version"] == "0.2.0.dev0" + assert next(iter(parsed)) == "version" assert "semantic_model" not in parsed assert parsed["name"] == "my_project" diff --git a/converters/orionbelt/src/ossie_orionbelt/validation.py b/converters/orionbelt/src/ossie_orionbelt/validation.py index b4dbe6f1..65d5af61 100644 --- a/converters/orionbelt/src/ossie_orionbelt/validation.py +++ b/converters/orionbelt/src/ossie_orionbelt/validation.py @@ -207,13 +207,19 @@ def validate_ossie(ossie_dict: dict[str, Any], schema_path: Path | None = None) _validate_json_schema(ossie_dict, schema_path or _OSSIE_SCHEMA_PATH, result, draft="draft2020") # The semantic checks below assume a well-formed structure (lists of dicts). - # JSON Schema validation above already reports structural errors, so guard - # every level here rather than raising on malformed input. + # Guard every level rather than raising on malformed input, even when + # JSON Schema validation is unavailable. def _as_dict_list(value: Any) -> list[dict[str, Any]]: return [item for item in value if isinstance(item, dict)] if isinstance(value, list) else [] - # Legacy wrappers are schema errors, not model contents to traverse. - if not isinstance(ossie_dict, dict) or "semantic_model" in ossie_dict: + if not isinstance(ossie_dict, dict): + result.semantic_errors.append("[INVALID_DOCUMENT] Ossie document must be an object") + return result + if "semantic_model" in ossie_dict: + result.semantic_errors.append( + "[LEGACY_WRAPPER] Legacy 'semantic_model' wrappers are not supported; " + "place model properties at the document root" + ) return result model = ossie_dict diff --git a/converters/orionbelt/tests/test_ossie_validation.py b/converters/orionbelt/tests/test_ossie_validation.py new file mode 100644 index 00000000..60c2d5d6 --- /dev/null +++ b/converters/orionbelt/tests/test_ossie_validation.py @@ -0,0 +1,67 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import sys + +import pytest + +from ossie_orionbelt.validation import _OSSIE_SCHEMA_PATH, validate_ossie + + +@pytest.fixture(params=["available", "missing_file", "missing_package"]) +def schema_path(request, tmp_path, monkeypatch): + if request.param == "missing_file": + return tmp_path / "missing-schema.json" + if request.param == "missing_package": + monkeypatch.setitem(sys.modules, "jsonschema", None) + return _OSSIE_SCHEMA_PATH + + +@pytest.mark.parametrize( + "wrapper", + [None, [], {}, [{"name": "legacy", "datasets": []}]], +) +@pytest.mark.parametrize("include_root_model", [False, True]) +def test_legacy_wrapper_is_always_invalid(schema_path, wrapper, include_root_model): + document = {"version": "0.2.0.dev0", "semantic_model": wrapper} + if include_root_model: + document.update(name="m", datasets=[{"name": "t", "source": "a.b.c"}]) + + result = validate_ossie(document, schema_path=schema_path) + + assert not result.valid + assert any("[LEGACY_WRAPPER]" in error for error in result.semantic_errors) + + +@pytest.mark.parametrize("document", [None, [], "not a mapping", 42]) +def test_non_mapping_document_is_always_invalid(schema_path, document): + result = validate_ossie(document, schema_path=schema_path) + + assert not result.valid + assert any("[INVALID_DOCUMENT]" in error for error in result.semantic_errors) + + +def test_valid_flat_document_can_be_checked_without_schema(schema_path): + document = { + "version": "0.2.0.dev0", + "name": "m", + "datasets": [{"name": "t", "source": "a.b.c"}], + } + + result = validate_ossie(document, schema_path=schema_path) + + assert result.valid diff --git a/converters/sigma/tests/test_roundtrip.py b/converters/sigma/tests/test_roundtrip.py index 6c03251d..078d836d 100644 --- a/converters/sigma/tests/test_roundtrip.py +++ b/converters/sigma/tests/test_roundtrip.py @@ -39,6 +39,7 @@ def test_sigma_osi_sigma_roundtrip_through_yaml_serialization(fixture_name): yaml_text = document.to_ossie_yaml() serialized = yaml.safe_load(yaml_text) + assert next(iter(serialized)) == "version" assert serialized["name"] == spec["name"] assert "semantic_model" not in serialized assert "dialects" not in serialized diff --git a/converters/wisdom/tests/test_wisdom_to_ossie.py b/converters/wisdom/tests/test_wisdom_to_ossie.py index e08c4d9f..9526689a 100644 --- a/converters/wisdom/tests/test_wisdom_to_ossie.py +++ b/converters/wisdom/tests/test_wisdom_to_ossie.py @@ -158,6 +158,7 @@ def test_stale_measure_is_kept_with_warning(result, model): def test_output_round_trips_through_ossie_yaml(result): serialized = yaml.safe_load(result.output.to_ossie_yaml()) + assert next(iter(serialized)) == "version" assert serialized["name"] == "Sample Sales" assert "semantic_model" not in serialized document = OssieDocument.model_validate(serialized) diff --git a/python/README.md b/python/README.md index 43f9e73b..02fd5a67 100644 --- a/python/README.md +++ b/python/README.md @@ -25,7 +25,8 @@ Each `OssieDocument` is one semantic model: `name`, `datasets`, `relationships`, and `metrics` sit at the root alongside `version`. Construct documents with `OssieDocument(name="sales", datasets=[...])` and access their datasets as `document.datasets`. JSON and YAML serialization use the same -flat shape. The former `semantic_model` wrapper is rejected. Unwrap old +flat shape, with `version` serialized first when included. The former +`semantic_model` wrapper is rejected. Unwrap old single-model documents and split multi-model documents into separate files, preserving model contents and setting `version` in each file before loading them. diff --git a/python/src/ossie/models.py b/python/src/ossie/models.py index a1825557..719356b1 100644 --- a/python/src/ossie/models.py +++ b/python/src/ossie/models.py @@ -19,7 +19,7 @@ from typing import Any, Optional, Union import yaml -from pydantic import BaseModel, ConfigDict, Field +from pydantic import BaseModel, ConfigDict, Field, SerializerFunctionWrapHandler, model_serializer class OssieDialect(str, Enum): @@ -214,6 +214,14 @@ class OssieDocument(OssieSemanticModel): version: str = "0.2.0.dev0" + @model_serializer(mode="wrap") + def _serialize_document(self, handler: SerializerFunctionWrapHandler): + # Omit a custom return type to retain Pydantic's serialization schema. + data = handler(self) + if "version" in data: + return {"version": data.pop("version"), **data} + return data + def to_ossie_yaml(self, **kwargs: Any) -> str: """Serialize to Ossie-compliant YAML (uses field aliases and excludes None values).""" data = self.model_dump(by_alias=True, exclude_none=True, mode="json", **kwargs) diff --git a/python/tests/test_models.py b/python/tests/test_models.py index 1a115903..46f9967d 100644 --- a/python/tests/test_models.py +++ b/python/tests/test_models.py @@ -123,6 +123,56 @@ def test_document_serialization_preserves_flat_model_and_metadata() -> None: assert OssieDocument.model_validate(serialized) == document +@pytest.mark.parametrize( + "serialize", ["model_dump", "model_dump_json", "to_ossie_yaml", "to_ossie_json"] +) +def test_document_serialization_puts_version_first(serialize: str) -> None: + data = _document() + document = OssieDocument.model_validate(data) + + serialized = getattr(document, serialize)() + if isinstance(serialized, str): + serialized = yaml.safe_load(serialized) + + assert next(iter(serialized)) == "version" + assert serialized["version"] == data["version"] + + +@pytest.mark.parametrize( + "serialize", ["model_dump", "model_dump_json", "to_ossie_yaml", "to_ossie_json"] +) +@pytest.mark.parametrize( + "options", + [ + {"exclude": {"version"}}, + {"include": {"name", "datasets"}}, + {"exclude_defaults": True}, + {"exclude_unset": True}, + ], +) +def test_document_serialization_can_omit_version(serialize: str, options: dict) -> None: + data = _document() + del data["version"] + document = OssieDocument.model_validate(data) + + serialized = getattr(document, serialize)(**options) + if isinstance(serialized, str): + serialized = yaml.safe_load(serialized) + + assert "version" not in serialized + assert serialized["name"] == data["name"] + + +def test_document_serialization_schema_preserves_model_fields() -> None: + schema = OssieDocument.model_json_schema(mode="serialization") + + assert schema["properties"]["version"]["type"] == "string" + assert schema["properties"]["datasets"]["type"] == "array" + # Early Pydantic 2.x versions also require defaulted fields in serialization schemas. + assert {"name", "datasets"} <= set(schema["required"]) + assert schema["additionalProperties"] is False + + @pytest.mark.parametrize( "legacy_value", [