Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions converters/dbt/tests/__snapshots__/test_msi_to_ossie.ambr
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
# under the License.
# name: TestMetricConversion.test_derived_metric_nested
'''
version: 0.2.0.dev0
name: semantic_model
datasets:
- name: orders
Expand Down Expand Up @@ -63,12 +64,12 @@
dialects:
- dialect: ANSI_SQL
expression: (SUM(orders.amount) - SUM(orders.cost_amount)) - SUM(orders.expense_amount)
version: 0.2.0.dev0

'''
# ---
# name: TestMetricConversion.test_ratio_metric_inlines_sub_expressions
'''
version: 0.2.0.dev0
name: semantic_model
datasets:
- name: orders
Expand Down Expand Up @@ -101,12 +102,12 @@
- dialect: ANSI_SQL
expression: (SUM(orders.amount)) / (SUM(CASE WHEN orders.order_id IS NOT NULL
THEN 1 ELSE 0 END))
version: 0.2.0.dev0

'''
# ---
# name: TestMetricFilterFlattening.test_metric_and_measure_filters_combined_with_and
'''
version: 0.2.0.dev0
name: semantic_model
datasets:
- name: orders
Expand All @@ -124,12 +125,12 @@
- dialect: ANSI_SQL
expression: SUM(CASE WHEN (status = 'paid') AND (region = 'intl') THEN orders.amount
END)
version: 0.2.0.dev0

'''
# ---
# name: TestRelationshipConversion.test_three_datasets_produce_all_pairs
'''
version: 0.2.0.dev0
name: semantic_model
datasets:
- name: users_a
Expand Down Expand Up @@ -182,7 +183,6 @@
- user_id
to_columns:
- user_id
version: 0.2.0.dev0

'''
# ---
2 changes: 1 addition & 1 deletion converters/dbt/tests/__snapshots__/test_ossie_to_msi.ambr
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
# under the License.
# name: TestOssieToMSIRoundTrip.test_ossie_to_msi_to_ossie_preserves_structure
'''
version: 0.2.0.dev0
name: semantic_model
datasets:
- name: orders
Expand Down Expand Up @@ -56,7 +57,6 @@
dialects:
- dialect: ANSI_SQL
expression: SUM(orders.amount)
version: 0.2.0.dev0

'''
# ---
1 change: 1 addition & 0 deletions converters/dbt/tests/test_msi_to_ossie.py
Original file line number Diff line number Diff line change
Expand Up @@ -1209,6 +1209,7 @@ def test_to_ossie_json_produces_valid_json(self) -> None:
parsed = json.loads(result.to_ossie_json())

assert parsed["version"] == "0.2.0.dev0"
assert next(iter(parsed)) == "version"
assert "semantic_model" not in parsed
assert parsed["name"] == "my_project"

Expand Down
14 changes: 10 additions & 4 deletions converters/orionbelt/src/ossie_orionbelt/validation.py
Original file line number Diff line number Diff line change
Expand Up @@ -207,13 +207,19 @@ def validate_ossie(ossie_dict: dict[str, Any], schema_path: Path | None = None)
_validate_json_schema(ossie_dict, schema_path or _OSSIE_SCHEMA_PATH, result, draft="draft2020")

# The semantic checks below assume a well-formed structure (lists of dicts).
# JSON Schema validation above already reports structural errors, so guard
# every level here rather than raising on malformed input.
# Guard every level rather than raising on malformed input, even when
# JSON Schema validation is unavailable.
def _as_dict_list(value: Any) -> list[dict[str, Any]]:
return [item for item in value if isinstance(item, dict)] if isinstance(value, list) else []

# Legacy wrappers are schema errors, not model contents to traverse.
if not isinstance(ossie_dict, dict) or "semantic_model" in ossie_dict:
if not isinstance(ossie_dict, dict):

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This fallback only catches two specific malformations (ossie_dict not a dict, or a semantic_model wrapper).
When jsonschema is unavailable (or schema_path points at a missing schema), every other structurally invalid document silently comes back as valid=True with empty schema_errors/semantic_errors. For instance datasets as a string instead of a list, datasets missing entirely, name missing, or a dataset dict missing its required source field.

Since _as_dict_list() already silently drops non-dict items instead of erroring, and the downstream unique-name/reference checks only ever see whatever survives that filter, none of these malformations get flagged anywhere in this fallback.

Given this function docstring says it mirrors validate.py three layer validation, could we add guards at least the missing/malformed datasets and missing name cases here too? Or explicitly document that JSON-Schema-unavailable mode is a reduced-coverage fallback and only guarantees the two checks above?

I believe it's currently confusing because it reads as full validation but it isn't when jsonschema is absent.

result.semantic_errors.append("[INVALID_DOCUMENT] Ossie document must be an object")
return result
if "semantic_model" in ossie_dict:
result.semantic_errors.append(
"[LEGACY_WRAPPER] Legacy 'semantic_model' wrappers are not supported; "
"place model properties at the document root"
)
return result
model = ossie_dict

Expand Down
67 changes: 67 additions & 0 deletions converters/orionbelt/tests/test_ossie_validation.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.

import sys

import pytest

from ossie_orionbelt.validation import _OSSIE_SCHEMA_PATH, validate_ossie


@pytest.fixture(params=["available", "missing_file", "missing_package"])
def schema_path(request, tmp_path, monkeypatch):
if request.param == "missing_file":
return tmp_path / "missing-schema.json"
if request.param == "missing_package":
monkeypatch.setitem(sys.modules, "jsonschema", None)
return _OSSIE_SCHEMA_PATH


@pytest.mark.parametrize(
"wrapper",
[None, [], {}, [{"name": "legacy", "datasets": []}]],
)
@pytest.mark.parametrize("include_root_model", [False, True])
def test_legacy_wrapper_is_always_invalid(schema_path, wrapper, include_root_model):
document = {"version": "0.2.0.dev0", "semantic_model": wrapper}
if include_root_model:
document.update(name="m", datasets=[{"name": "t", "source": "a.b.c"}])

result = validate_ossie(document, schema_path=schema_path)

assert not result.valid
assert any("[LEGACY_WRAPPER]" in error for error in result.semantic_errors)


@pytest.mark.parametrize("document", [None, [], "not a mapping", 42])
def test_non_mapping_document_is_always_invalid(schema_path, document):
result = validate_ossie(document, schema_path=schema_path)

assert not result.valid
assert any("[INVALID_DOCUMENT]" in error for error in result.semantic_errors)


def test_valid_flat_document_can_be_checked_without_schema(schema_path):
document = {
"version": "0.2.0.dev0",
"name": "m",
"datasets": [{"name": "t", "source": "a.b.c"}],
}

result = validate_ossie(document, schema_path=schema_path)

assert result.valid
1 change: 1 addition & 0 deletions converters/sigma/tests/test_roundtrip.py
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ def test_sigma_osi_sigma_roundtrip_through_yaml_serialization(fixture_name):
yaml_text = document.to_ossie_yaml()

serialized = yaml.safe_load(yaml_text)
assert next(iter(serialized)) == "version"
assert serialized["name"] == spec["name"]
assert "semantic_model" not in serialized
assert "dialects" not in serialized
Expand Down
1 change: 1 addition & 0 deletions converters/wisdom/tests/test_wisdom_to_ossie.py
Original file line number Diff line number Diff line change
Expand Up @@ -158,6 +158,7 @@ def test_stale_measure_is_kept_with_warning(result, model):

def test_output_round_trips_through_ossie_yaml(result):
serialized = yaml.safe_load(result.output.to_ossie_yaml())
assert next(iter(serialized)) == "version"
assert serialized["name"] == "Sample Sales"
assert "semantic_model" not in serialized
document = OssieDocument.model_validate(serialized)
Expand Down
3 changes: 2 additions & 1 deletion python/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,8 @@ Each `OssieDocument` is one semantic model: `name`, `datasets`, `relationships`,
and `metrics` sit at the root alongside `version`.
Construct documents with `OssieDocument(name="sales", datasets=[...])` and access
their datasets as `document.datasets`. JSON and YAML serialization use the same
flat shape. The former `semantic_model` wrapper is rejected. Unwrap old
flat shape, with `version` serialized first when included. The former
`semantic_model` wrapper is rejected. Unwrap old
single-model documents and split multi-model documents into separate files,
preserving model contents and setting `version` in each file before loading them.

Expand Down
10 changes: 9 additions & 1 deletion python/src/ossie/models.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@
from typing import Any, Optional, Union

import yaml
from pydantic import BaseModel, ConfigDict, Field
from pydantic import BaseModel, ConfigDict, Field, SerializerFunctionWrapHandler, model_serializer


class OssieDialect(str, Enum):
Expand Down Expand Up @@ -214,6 +214,14 @@ class OssieDocument(OssieSemanticModel):

version: str = "0.2.0.dev0"

@model_serializer(mode="wrap")
def _serialize_document(self, handler: SerializerFunctionWrapHandler):
# Omit a custom return type to retain Pydantic's serialization schema.
data = handler(self)
if "version" in data:
return {"version": data.pop("version"), **data}
return data

def to_ossie_yaml(self, **kwargs: Any) -> str:
"""Serialize to Ossie-compliant YAML (uses field aliases and excludes None values)."""
data = self.model_dump(by_alias=True, exclude_none=True, mode="json", **kwargs)
Expand Down
50 changes: 50 additions & 0 deletions python/tests/test_models.py
Original file line number Diff line number Diff line change
Expand Up @@ -123,6 +123,56 @@ def test_document_serialization_preserves_flat_model_and_metadata() -> None:
assert OssieDocument.model_validate(serialized) == document


@pytest.mark.parametrize(
"serialize", ["model_dump", "model_dump_json", "to_ossie_yaml", "to_ossie_json"]
)
def test_document_serialization_puts_version_first(serialize: str) -> None:
data = _document()
document = OssieDocument.model_validate(data)

serialized = getattr(document, serialize)()
if isinstance(serialized, str):
serialized = yaml.safe_load(serialized)

assert next(iter(serialized)) == "version"
assert serialized["version"] == data["version"]


@pytest.mark.parametrize(
"serialize", ["model_dump", "model_dump_json", "to_ossie_yaml", "to_ossie_json"]
)
@pytest.mark.parametrize(
"options",
[
{"exclude": {"version"}},
{"include": {"name", "datasets"}},
{"exclude_defaults": True},
{"exclude_unset": True},
],
)
def test_document_serialization_can_omit_version(serialize: str, options: dict) -> None:
data = _document()
del data["version"]
document = OssieDocument.model_validate(data)

serialized = getattr(document, serialize)(**options)
if isinstance(serialized, str):
serialized = yaml.safe_load(serialized)

assert "version" not in serialized
assert serialized["name"] == data["name"]


def test_document_serialization_schema_preserves_model_fields() -> None:
schema = OssieDocument.model_json_schema(mode="serialization")

assert schema["properties"]["version"]["type"] == "string"
assert schema["properties"]["datasets"]["type"] == "array"
# Early Pydantic 2.x versions also require defaulted fields in serialization schemas.
assert {"name", "datasets"} <= set(schema["required"])
assert schema["additionalProperties"] is False


@pytest.mark.parametrize(
"legacy_value",
[
Expand Down
Loading