From f07d32325d483b4f53cf6ae8a1db4e8cf6b8313a Mon Sep 17 00:00:00 2001 From: Ralf Becher Date: Thu, 17 Sep 2026 08:26:09 +0200 Subject: [PATCH 1/2] fix(orionbelt): carry field and metric datatype in both directions Closes #409. The OrionBelt converter ignored the spec `datatype` on import (it read the non-spec `data_type` key against a lowercase map) and never wrote it on export, so other Ossie tools saw wrong or missing logical types. - Ossie -> OBML: field `datatype` maps to OBML `abstractType` (precedence: `datatype` > legacy `data_type` > name heuristic). `Decimal` narrows to `float`, `Opaque` falls back to the heuristic. The stashed `obml_abstract_type` is now restored, so OBML-origin round trips stay exact. - Ossie -> OBML: metric `datatype` maps to the exact measure/metric `dataType` (`Decimal` -> `decimal(18, 2)`), unless one was restored from the extension. - OBML -> Ossie: fields always emit `datatype`; measures/metrics emit it only when an explicit `dataType` is declared, keeping round trips idempotent. Ported from the downstream osi-orionbelt package, where this shipped in ralforion/orionbelt-semantic-layer#246. --- .../orionbelt/ossie_obml_mapping_analysis.md | 5 +- .../orionbelt/src/ossie_orionbelt/_common.py | 87 +++++++ .../src/ossie_orionbelt/obml_to_ossie.py | 22 +- .../src/ossie_orionbelt/ossie_to_obml.py | 31 ++- .../tests/test_ossie_converter_datatype.py | 238 ++++++++++++++++++ .../tests/test_ossie_metric_no_silent_loss.py | 12 + 6 files changed, 388 insertions(+), 7 deletions(-) create mode 100644 converters/orionbelt/tests/test_ossie_converter_datatype.py diff --git a/converters/orionbelt/ossie_obml_mapping_analysis.md b/converters/orionbelt/ossie_obml_mapping_analysis.md index 957e9cf0..f835589a 100644 --- a/converters/orionbelt/ossie_obml_mapping_analysis.md +++ b/converters/orionbelt/ossie_obml_mapping_analysis.md @@ -143,7 +143,8 @@ These OBML features have no direct Ossie equivalent. Where possible, metadata is - Measure `withinGroup` — preserved in metric `custom_extensions` (`obml_within_group`) - Metric `format` — preserved in metric `custom_extensions` (`obml_format`) - Locale settings — not yet preserved -- `abstractType` (OBML type system) — preserved in field `custom_extensions` (`obml_abstract_type`) +- `abstractType` (OBML type system): emitted as the spec field `datatype` (`json` → `Opaque`, `time_tz` → `Time`) and preserved exactly in field `custom_extensions` (`obml_abstract_type`) +- Measure/metric `dataType`: emitted as metric `datatype` when declared (`decimal(p, s)` → `Decimal`), exact value preserved via `obml_data_type` ### 2.6 Ossie-Specific Features and How They Map to OBML @@ -158,7 +159,7 @@ These OBML features have no direct Ossie equivalent. Where possible, metadata is ### 3.1 Ossie → OBML 1. Parse `source` string to extract `database`, `schema`, and `table` -2. Convert fields to columns with type inference (heuristic-based `abstractType`) +2. Convert fields to columns; `abstractType` comes from the spec `datatype` (`Decimal` narrows to `float`), then legacy `data_type`, then a name heuristic (also used for `Opaque`); metric `datatype` sets the exact measure/metric `dataType` (`Decimal` → `decimal(18, 2)`) 3. Restructure global relationships into inline joins on data objects 4. Decompose metric SQL expressions into OBML measures + metrics 5. Extract dimension-flagged fields into the top-level `dimensions` section (excluding FK/PK join keys) diff --git a/converters/orionbelt/src/ossie_orionbelt/_common.py b/converters/orionbelt/src/ossie_orionbelt/_common.py index 9d4a5412..9e189291 100644 --- a/converters/orionbelt/src/ossie_orionbelt/_common.py +++ b/converters/orionbelt/src/ossie_orionbelt/_common.py @@ -91,3 +91,90 @@ "timestamp": "timestamp", "boolean": "boolean", } + +# ─── Ossie DataType ⇄ OBML ────────────────────────────────────────────────── +# Ossie `datatype` on Field/Metric is a *logical* type backed by the capitalised +# `DataType` enum in core-spec/ossie-schema.json - the same layer as OBML's column +# `abstractType` - so this is the field/dimension mapping. +# +# `Decimal` has no logical-layer equivalent in OBML: OBML models exact decimal at +# the physical/result layer (`sqlType`/`sqlPrecision`/`sqlScale`, measure/metric +# `dataType` via `decimal(p, s)`), not as a coarse `abstractType`. So `Decimal` +# narrows to `float` for fields, but is recovered exactly for metrics via the +# physical `dataType` map below (`OSSIE_DATATYPE_TO_OBML_PHYSICAL`). +# +# `Opaque` is Ossie's "known type outside the portable vocabulary" marker and is +# intentionally absent so it falls back to the name heuristic on import. +OSSIE_DATATYPE_TO_OBML_ABSTRACT = { + "String": "string", + "Integer": "int", + "Float": "float", + "Decimal": "float", + "Boolean": "boolean", + "Date": "date", + "Time": "time", + "DateTime": "timestamp", + "DateTimeTz": "timestamp_tz", +} + +# OBML column `abstractType` -> Ossie `DataType`, for the export direction. +OBML_ABSTRACT_TO_OSSIE_DATATYPE = { + "string": "String", + "json": "Opaque", + "int": "Integer", + "float": "Float", + "date": "Date", + "time": "Time", + "time_tz": "Time", + "timestamp": "DateTime", + "timestamp_tz": "DateTimeTz", + "boolean": "Boolean", +} + +# Metric/measure `datatype`. Unlike fields, OBML measures/metrics carry an exact +# `dataType` (physical vocabulary: `integer`/`double`/`decimal(p, s)`/...), which +# is where `Decimal` genuinely belongs. So Ossie metric `datatype` maps to that +# field, not the coarse `abstractType`. +OBML_DECIMAL_DEFAULT = "decimal(18, 2)" # mirrors OrionBelt's built-in default + +# Ossie `DataType` -> OBML physical `dataType` (import direction). `Opaque` is +# omitted (non-portable). `DateTimeTz` has no tz-aware physical form, so it +# narrows to `timestamp`. +OSSIE_DATATYPE_TO_OBML_PHYSICAL = { + "String": "string", + "Integer": "integer", + "Float": "double", + "Decimal": OBML_DECIMAL_DEFAULT, + "Boolean": "boolean", + "Date": "date", + "Time": "time", + "DateTime": "timestamp", + "DateTimeTz": "timestamp", +} + +# OBML physical `dataType` -> Ossie `DataType` (export direction). `decimal(p, s)` +# is handled by ``obml_datatype_to_ossie`` since it is parametrised. +OBML_PHYSICAL_TO_OSSIE_DATATYPE = { + "string": "String", + "integer": "Integer", + "bigint": "Integer", + "double": "Float", + "boolean": "Boolean", + "date": "Date", + "time": "Time", + "timestamp": "DateTime", +} + + +def obml_datatype_to_ossie(data_type: str | None) -> str | None: + """Map an explicit OBML measure/metric ``dataType`` to an Ossie ``DataType``. + + Returns ``None`` when there is no mapping, so the caller emits nothing rather + than an unknown type. ``decimal(p, s)`` maps to ``Decimal``. + """ + if not data_type: + return None + normalized = data_type.strip().lower() + if normalized.startswith("decimal"): + return "Decimal" + return OBML_PHYSICAL_TO_OSSIE_DATATYPE.get(normalized) diff --git a/converters/orionbelt/src/ossie_orionbelt/obml_to_ossie.py b/converters/orionbelt/src/ossie_orionbelt/obml_to_ossie.py index d46687eb..1990f719 100644 --- a/converters/orionbelt/src/ossie_orionbelt/obml_to_ossie.py +++ b/converters/orionbelt/src/ossie_orionbelt/obml_to_ossie.py @@ -32,7 +32,9 @@ _OSSIE_VENDOR_READ, _OSSIE_VERSION, _VENDOR_OBML, + OBML_ABSTRACT_TO_OSSIE_DATATYPE, OBML_TO_OSSIE_TYPE, + obml_datatype_to_ossie, ) @@ -385,8 +387,12 @@ def _convert_column( if ai_ctx: field["ai_context"] = ai_ctx - # Preserve OBML type info in custom_extensions for roundtrip fidelity + # Emit the spec `datatype` from the OBML abstractType so exported fields + # carry a portable logical type... abstract_type = col_obj.get("abstractType", "string") + field["datatype"] = OBML_ABSTRACT_TO_OSSIE_DATATYPE.get(abstract_type, "String") + # ...and stash the exact abstractType in custom_extensions so the return + # trip restores it verbatim, lossless through the narrowing map. ossie_type = OBML_TO_OSSIE_TYPE.get(abstract_type, "string") ext_data: dict[str, Any] = { "data_type": ossie_type, @@ -549,6 +555,7 @@ def _convert_measures_and_metrics( ossie_metric = self._convert_measure(measure_name, measure_obj, data_objects) if ossie_metric: self._carry_foreign_to_ossie_metric(measure_obj, ossie_metric) + self._emit_ossie_metric_datatype(measure_obj, ossie_metric) ossie_metrics.append(ossie_metric) # Convert OBML metrics (which reference measures) to Ossie metrics @@ -571,6 +578,7 @@ def _convert_measures_and_metrics( ) if ossie_metric: self._carry_foreign_to_ossie_metric(metric_obj, ossie_metric) + self._emit_ossie_metric_datatype(metric_obj, ossie_metric) ossie_metrics.append(ossie_metric) return ossie_metrics @@ -584,6 +592,18 @@ def _carry_foreign_to_ossie_metric(self, obml_obj: dict, ossie_metric: dict) -> if not ossie_metric["custom_extensions"]: del ossie_metric["custom_extensions"] + def _emit_ossie_metric_datatype(self, obml_obj: dict, ossie_metric: dict) -> None: + """Emit the spec `datatype` from an explicit OBML measure/metric `dataType`. + + Only fires when the OBML object declares an exact `dataType`, so plain + measures (whose type is only the defaulted `resultType`) stay untouched + and round trips stay idempotent. The exact `dataType` also round-trips via + `obml_data_type` in `custom_extensions`; this adds the portable field. + """ + ossie_dt = obml_datatype_to_ossie(obml_obj.get("dataType")) + if ossie_dt: + ossie_metric["datatype"] = ossie_dt + def _convert_measure(self, name: str, measure: dict, data_objects: dict) -> dict | None: """Convert an OBML measure to an Ossie metric.""" diff --git a/converters/orionbelt/src/ossie_orionbelt/ossie_to_obml.py b/converters/orionbelt/src/ossie_orionbelt/ossie_to_obml.py index 600d38cb..7c04b334 100644 --- a/converters/orionbelt/src/ossie_orionbelt/ossie_to_obml.py +++ b/converters/orionbelt/src/ossie_orionbelt/ossie_to_obml.py @@ -34,6 +34,8 @@ _OSSIE_VERSION, _SQL_PARSEABLE_DIALECTS, _VENDOR_OSSIE, + OSSIE_DATATYPE_TO_OBML_ABSTRACT, + OSSIE_DATATYPE_TO_OBML_PHYSICAL, OSSIE_TO_OBML_TYPE, ) @@ -374,10 +376,17 @@ def _convert_field(self, field: dict) -> tuple[str, dict]: elif code == name and dialects: code = dialects[0].get("expression", name) - # Determine abstract type: prefer explicit data_type, fall back to heuristic - ossie_type = field.get("data_type", "") - if ossie_type and ossie_type in OSSIE_TO_OBML_TYPE: - abstract_type = OSSIE_TO_OBML_TYPE[ossie_type] + # Determine abstract type. Precedence: the spec `datatype` (capitalised + # `DataType` enum) > legacy lowercase `data_type` > name heuristic. An + # OBML-origin field additionally restores its exact `abstractType` from + # the stashed extension below (highest precedence), keeping + # OBML -> Ossie -> OBML lossless. + ossie_datatype = field.get("datatype", "") + legacy_type = field.get("data_type", "") + if ossie_datatype in OSSIE_DATATYPE_TO_OBML_ABSTRACT: + abstract_type = OSSIE_DATATYPE_TO_OBML_ABSTRACT[ossie_datatype] + elif legacy_type and legacy_type in OSSIE_TO_OBML_TYPE: + abstract_type = OSSIE_TO_OBML_TYPE[legacy_type] else: abstract_type = self._infer_obml_type(field) @@ -411,6 +420,11 @@ def _convert_field(self, field: dict) -> tuple[str, dict]: if ext.get("vendor_name") in _OBML_VENDOR_READ: try: ext_data = json.loads(ext.get("data", "{}")) + # Restore the exact OBML abstractType stashed on export, so a + # narrowing datatype map (e.g. Decimal -> float) never + # degrades an OBML-origin round trip. + if ext_data.get("obml_abstract_type"): + col["abstractType"] = ext_data["obml_abstract_type"] if ext_data.get("obml_sql_type"): col["sqlType"] = ext_data["obml_sql_type"] if ext_data.get("obml_sql_precision") is not None: @@ -829,6 +843,15 @@ def _convert_metrics(self, ossie_metrics: list, ds_map: dict) -> tuple[dict, dic target = metrics.get(m["name"]) or measures.get(m["name"]) if target is not None: self._carry_foreign_extensions(m.get("custom_extensions"), target) + # Ossie metric `datatype` -> OBML exact `dataType` (its natural + # home; `Decimal` -> decimal(p, s)). Don't override a dataType + # already restored from an OBML-origin extension, and skip + # Opaque/unknown (absent from the map). + ossie_dt = m.get("datatype") + if ossie_dt and not target.get("dataType"): + obml_dt = OSSIE_DATATYPE_TO_OBML_PHYSICAL.get(ossie_dt) + if obml_dt: + target["dataType"] = obml_dt return measures, metrics diff --git a/converters/orionbelt/tests/test_ossie_converter_datatype.py b/converters/orionbelt/tests/test_ossie_converter_datatype.py new file mode 100644 index 00000000..681d9ee7 --- /dev/null +++ b/converters/orionbelt/tests/test_ossie_converter_datatype.py @@ -0,0 +1,238 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""Ossie ``datatype`` support on the field conversion path. + +Ossie defines a first-class ``datatype`` on ``Field``/``Metric`` backed by a +capitalised ``DataType`` enum. The converter reads it (over the name heuristic) +on import and emits it on export. Because OBML's ``abstractType`` is a coarse +*logical* layer with no exact ``decimal``, ``Decimal`` narrows to ``float`` for +fields - so the exact ``abstractType`` is stashed for a lossless return trip. +""" + +from __future__ import annotations + +from typing import Any + +import ossie_orionbelt.converter as conv + + +def _ossie_field(name: str, **extra: Any) -> dict[str, Any]: + field: dict[str, Any] = { + "name": name, + "expression": {"dialects": [{"dialect": "ANSI_SQL", "expression": name}]}, + } + field.update(extra) + return field + + +def _ossie_model(fields: list[dict[str, Any]]) -> dict[str, Any]: + return { + "version": "0.2.0.dev0", + "semantic_model": [ + { + "name": "sales", + "datasets": [ + {"name": "Orders", "source": "ANALYTICS.PUBLIC.ORDERS", "fields": fields} + ], + } + ], + } + + +def _obml_columns(obml: dict[str, Any]) -> dict[str, dict[str, Any]]: + (obj,) = obml["dataObjects"].values() + return obj["columns"] + + +def _ossie_fields(ossie: dict[str, Any]) -> dict[str, dict[str, Any]]: + out: dict[str, dict[str, Any]] = {} + for ds in ossie["semantic_model"][0]["datasets"]: + for f in ds.get("fields", []): + out[f["name"]] = f + return out + + +class TestImportDatatype: + """Ossie ``datatype`` maps to OBML ``abstractType`` on import.""" + + def test_direct_mappings(self) -> None: + ossie = _ossie_model( + [ + _ossie_field("s", datatype="String"), + _ossie_field("i", datatype="Integer"), + _ossie_field("f", datatype="Float"), + _ossie_field("b", datatype="Boolean"), + _ossie_field("d", datatype="Date"), + _ossie_field("t", datatype="Time"), + _ossie_field("dt", datatype="DateTime"), + _ossie_field("dttz", datatype="DateTimeTz"), + ] + ) + cols = _obml_columns(conv.OssietoOBML(ossie).convert()) + got = {name: c["abstractType"] for name, c in cols.items()} + assert got == { + "s": "string", + "i": "int", + "f": "float", + "b": "boolean", + "d": "date", + "t": "time", + "dt": "timestamp", + "dttz": "timestamp_tz", + } + + def test_decimal_narrows_to_float(self) -> None: + cols = _obml_columns( + conv.OssietoOBML(_ossie_model([_ossie_field("d", datatype="Decimal")])).convert() + ) + assert cols["d"]["abstractType"] == "float" + + def test_opaque_falls_back_to_heuristic(self) -> None: + # `price` is a heuristic float keyword; Opaque must not override it to a + # literal type - it means "unknown/non-portable", so the heuristic runs. + cols = _obml_columns( + conv.OssietoOBML(_ossie_model([_ossie_field("price", datatype="Opaque")])).convert() + ) + assert cols["price"]["abstractType"] == "float" + + def test_datatype_wins_over_legacy_and_heuristic(self) -> None: + # New capitalised `datatype` beats the legacy lowercase `data_type` and + # the name heuristic (name `amount` would heuristically be float). + ossie = _ossie_model([_ossie_field("amount", datatype="Integer", data_type="number")]) + cols = _obml_columns(conv.OssietoOBML(ossie).convert()) + assert cols["amount"]["abstractType"] == "int" + + +class TestExportDatatype: + """OBML ``abstractType`` emits a first-class Ossie ``datatype`` on export.""" + + @staticmethod + def _obml(abstract_type: str) -> dict[str, Any]: + return { + "dataObjects": { + "Orders": { + "code": "orders", + "columns": {"Val": {"code": "val", "abstractType": abstract_type}}, + } + } + } + + def test_emits_capitalised_datatype(self) -> None: + for abstract_type, expected in [ + ("int", "Integer"), + ("float", "Float"), + ("timestamp_tz", "DateTimeTz"), + ("json", "Opaque"), + ("boolean", "Boolean"), + ]: + ossie = conv.OBMLtoOssie(self._obml(abstract_type), model_name="s").convert() + assert _ossie_fields(ossie)["val"]["datatype"] == expected + + +class TestRoundtripLossless: + """OBML -> Ossie -> OBML preserves the exact abstractType via the stash.""" + + def test_narrowing_types_survive(self) -> None: + # json -> Opaque and time_tz -> Time are lossy in the datatype map alone; + # the stashed obml_abstract_type must restore them exactly. + for abstract_type in ["json", "time_tz", "float", "timestamp_tz"]: + obml = { + "dataObjects": { + "Orders": { + "code": "orders", + "columns": {"Val": {"code": "val", "abstractType": abstract_type}}, + } + } + } + back = conv.OssietoOBML(conv.OBMLtoOssie(obml, model_name="s").convert()).convert() + # The column key round-trips to its code (`val`); assert on the + # single column's abstractType regardless of its restored key. + (col,) = _obml_columns(back).values() + assert col["abstractType"] == abstract_type + + +def _ossie_model_with_metric(datatype: str) -> dict[str, Any]: + return { + "version": "0.2.0.dev0", + "semantic_model": [ + { + "name": "sales", + "datasets": [ + { + "name": "Orders", + "source": "A.P.ORDERS", + "fields": [_ossie_field("amount")], + } + ], + "metrics": [ + { + "name": "Total", + "expression": { + "dialects": [ + {"dialect": "ANSI_SQL", "expression": "SUM(Orders.amount)"} + ] + }, + "datatype": datatype, + } + ], + } + ], + } + + +class TestMetricDatatype: + """Ossie metric ``datatype`` maps to the exact OBML ``dataType`` and round-trips.""" + + def test_import_sets_exact_data_type(self) -> None: + for ossie_dt, expected in [ + ("Decimal", "decimal(18, 2)"), + ("Integer", "integer"), + ("Float", "double"), + ]: + obml = conv.OssietoOBML(_ossie_model_with_metric(ossie_dt)).convert() + # SUM(Orders.amount) becomes an OBML measure named "Total". + assert obml["measures"]["Total"]["dataType"] == expected + + def test_roundtrip_preserves_metric_datatype(self) -> None: + # Regression: Ossie -> OBML -> Ossie used to drop the metric datatype. + for ossie_dt in ["Decimal", "Integer", "Float"]: + ossie = _ossie_model_with_metric(ossie_dt) + back = conv.OBMLtoOssie(conv.OssietoOBML(ossie).convert(), model_name="sales").convert() + metric = back["semantic_model"][0]["metrics"][0] + assert metric.get("datatype") == ossie_dt + + def test_plain_measure_emits_no_datatype(self) -> None: + # A measure with no explicit dataType (only the defaulted resultType) + # must not gain a datatype on export - keeps round trips idempotent. + obml = { + "dataObjects": { + "Orders": { + "code": "orders", + "columns": {"Amount": {"code": "amount", "abstractType": "float"}}, + } + }, + "measures": { + "Total": { + "columns": [{"dataObject": "Orders", "column": "Amount"}], + "resultType": "float", + "aggregation": "sum", + } + }, + } + ossie = conv.OBMLtoOssie(obml, model_name="s").convert() + metric = ossie["semantic_model"][0]["metrics"][0] + assert "datatype" not in metric diff --git a/converters/orionbelt/tests/test_ossie_metric_no_silent_loss.py b/converters/orionbelt/tests/test_ossie_metric_no_silent_loss.py index d3fb61d0..5c97410f 100644 --- a/converters/orionbelt/tests/test_ossie_metric_no_silent_loss.py +++ b/converters/orionbelt/tests/test_ossie_metric_no_silent_loss.py @@ -380,3 +380,15 @@ def _unconverted_stash(obml: dict[str, Any]) -> list[dict[str, Any]]: if "obml_unconverted_metrics" in data: return data["obml_unconverted_metrics"] return [] + + +def test_explicit_datatype_roundtrips() -> None: + """A field's spec `datatype` survives both directions (apache/ossie#409).""" + _, col = conv.OssietoOBML(ossie={})._convert_field( + {"name": "total_amount", "datatype": "String"} + ) + assert col["abstractType"] == "string" + field = conv.OBMLtoOssie(obml={})._convert_column( + "total_amount", {"code": "total_amount", "abstractType": "string"}, "Orders", {} + ) + assert field.get("datatype") == "String" From 7375c7ee564631d5ca47d0b7c3ff42d769e76f24 Mon Sep 17 00:00:00 2001 From: Ralf Becher Date: Thu, 17 Sep 2026 19:09:55 +0200 Subject: [PATCH 2/2] fix(orionbelt): address review of the datatype mapping - A non-string `dataType` in OBML, or a non-string field or metric `datatype` in Ossie, no longer raises: it has no mapping, the same as an unknown type, instead of aborting the conversion. - An Ossie `Decimal` metric takes the model's `settings.defaultNumericDataType` when the OBML-origin model carries one, and falls back to `decimal(18, 2)` otherwise. OrionBelt only accepts a `decimal(p, s)` there, so anything else falls back too. The settings are read ahead of the metrics; they are still restored after them. - A stashed `obml_data_type` or `obml_abstract_type` is kept only while it agrees with `datatype`. When it names a different type, `datatype` was edited in Ossie after the export and wins. A stash that agrees is kept, so `decimal(20, 6)` and `bigint` still round-trip exactly. Co-Authored-By: Claude Opus 5 (1M context) --- .../orionbelt/ossie_obml_mapping_analysis.md | 2 +- .../orionbelt/src/ossie_orionbelt/_common.py | 40 +++++++- .../src/ossie_orionbelt/ossie_to_obml.py | 78 ++++++++++++--- .../tests/test_ossie_converter_datatype.py | 97 +++++++++++++++++++ 4 files changed, 197 insertions(+), 20 deletions(-) diff --git a/converters/orionbelt/ossie_obml_mapping_analysis.md b/converters/orionbelt/ossie_obml_mapping_analysis.md index f835589a..17a97f06 100644 --- a/converters/orionbelt/ossie_obml_mapping_analysis.md +++ b/converters/orionbelt/ossie_obml_mapping_analysis.md @@ -159,7 +159,7 @@ These OBML features have no direct Ossie equivalent. Where possible, metadata is ### 3.1 Ossie → OBML 1. Parse `source` string to extract `database`, `schema`, and `table` -2. Convert fields to columns; `abstractType` comes from the spec `datatype` (`Decimal` narrows to `float`), then legacy `data_type`, then a name heuristic (also used for `Opaque`); metric `datatype` sets the exact measure/metric `dataType` (`Decimal` → `decimal(18, 2)`) +2. Convert fields to columns; `abstractType` comes from the spec `datatype` (`Decimal` narrows to `float`), then legacy `data_type`, then a name heuristic (also used for `Opaque`); metric `datatype` sets the exact measure/metric `dataType` (`Decimal` → the model's `settings.defaultNumericDataType`, else `decimal(18, 2)`). A stashed `obml_abstract_type` or `obml_data_type` is restored while it agrees with `datatype`; an edited `datatype` wins over it 3. Restructure global relationships into inline joins on data objects 4. Decompose metric SQL expressions into OBML measures + metrics 5. Extract dimension-flagged fields into the top-level `dimensions` section (excluding FK/PK join keys) diff --git a/converters/orionbelt/src/ossie_orionbelt/_common.py b/converters/orionbelt/src/ossie_orionbelt/_common.py index 9e189291..1987c384 100644 --- a/converters/orionbelt/src/ossie_orionbelt/_common.py +++ b/converters/orionbelt/src/ossie_orionbelt/_common.py @@ -166,15 +166,49 @@ } -def obml_datatype_to_ossie(data_type: str | None) -> str | None: +def obml_datatype_to_ossie(data_type: object) -> str | None: """Map an explicit OBML measure/metric ``dataType`` to an Ossie ``DataType``. Returns ``None`` when there is no mapping, so the caller emits nothing rather - than an unknown type. ``decimal(p, s)`` maps to ``Decimal``. + than an unknown type. ``decimal(p, s)`` maps to ``Decimal``. A hand-authored + document may carry a non-string ``dataType`` (``123``) that no schema check + has rejected yet; that has no mapping either, rather than aborting the whole + conversion. """ - if not data_type: + if not isinstance(data_type, str): return None normalized = data_type.strip().lower() + if not normalized: + return None if normalized.startswith("decimal"): return "Decimal" return OBML_PHYSICAL_TO_OSSIE_DATATYPE.get(normalized) + + +def obml_decimal_default(settings: object) -> str: + """The ``dataType`` an Ossie ``Decimal`` metric becomes in this model. + + OBML lets a model set ``settings.defaultNumericDataType`` (always a + ``decimal(p, s)``, which OrionBelt enforces), and a model configured for + ``decimal(20, 6)`` should not have its metrics written as the built-in + ``decimal(18, 2)``. Anything other than a decimal string there falls back to + the built-in default. + """ + if isinstance(settings, dict): + configured = settings.get("defaultNumericDataType") + if isinstance(configured, str) and configured.strip().lower().startswith("decimal"): + return configured + return OBML_DECIMAL_DEFAULT + + +def ossie_metric_datatype_to_obml(ossie_datatype: object, decimal_default: str) -> str | None: + """Map an Ossie metric ``datatype`` to the OBML measure/metric ``dataType``. + + ``Decimal`` takes the model's numeric default; ``Opaque``, an unknown value + or a non-string has no mapping. + """ + if not isinstance(ossie_datatype, str): + return None + if ossie_datatype == "Decimal": + return decimal_default + return OSSIE_DATATYPE_TO_OBML_PHYSICAL.get(ossie_datatype) diff --git a/converters/orionbelt/src/ossie_orionbelt/ossie_to_obml.py b/converters/orionbelt/src/ossie_orionbelt/ossie_to_obml.py index 7c04b334..b2cc5410 100644 --- a/converters/orionbelt/src/ossie_orionbelt/ossie_to_obml.py +++ b/converters/orionbelt/src/ossie_orionbelt/ossie_to_obml.py @@ -34,9 +34,13 @@ _OSSIE_VERSION, _SQL_PARSEABLE_DIALECTS, _VENDOR_OSSIE, + OBML_ABSTRACT_TO_OSSIE_DATATYPE, + OBML_DECIMAL_DEFAULT, OSSIE_DATATYPE_TO_OBML_ABSTRACT, - OSSIE_DATATYPE_TO_OBML_PHYSICAL, OSSIE_TO_OBML_TYPE, + obml_datatype_to_ossie, + obml_decimal_default, + ossie_metric_datatype_to_obml, ) # A dataset/column identifier in a resolved metric expression: either a bare SQL @@ -60,6 +64,10 @@ def __init__( # or an expression our parser cannot decompose). Preserved verbatim # rather than dropped — see ``_preserve_unconverted_metric``. self._unconverted_metrics: list[dict] = [] + # What an Ossie ``Decimal`` metric becomes: the model's own + # ``settings.defaultNumericDataType`` when it carries one, set per model + # in ``convert``. + self._decimal_default = OBML_DECIMAL_DEFAULT def _normalize_legacy_v01(self) -> None: """Promote Ossie v0.1.x payloads to the v0.2 shape, in place. @@ -170,6 +178,7 @@ def convert(self) -> dict: # ── Measures & Metrics ────────────────────────────────────── ossie_metrics = model.get("metrics", []) + self._decimal_default = obml_decimal_default(self._stashed_obml_settings(model)) measures, metrics = self._convert_metrics(ossie_metrics, ds_map) if measures: obml["measures"] = measures @@ -213,6 +222,22 @@ def convert(self) -> dict: return obml + @staticmethod + def _stashed_obml_settings(model: dict) -> object: + """The OBML ``settings`` an OBML-origin model stashed on export, if any. + + Read ahead of the metrics, which need the numeric default; the model + properties themselves are restored after them, as before. + """ + for ext in model.get("custom_extensions", []): + if ext.get("vendor_name") in _OBML_VENDOR_READ: + try: + data = json.loads(ext.get("data", "{}")) + except (json.JSONDecodeError, TypeError): + return None + return data.get("obml_settings") if isinstance(data, dict) else None + return None + @staticmethod def _carry_foreign_extensions(ossie_exts: list[dict] | None, obml_target: dict[str, Any]) -> None: """Carry third-party Ossie custom_extensions verbatim into OBML. @@ -379,10 +404,15 @@ def _convert_field(self, field: dict) -> tuple[str, dict]: # Determine abstract type. Precedence: the spec `datatype` (capitalised # `DataType` enum) > legacy lowercase `data_type` > name heuristic. An # OBML-origin field additionally restores its exact `abstractType` from - # the stashed extension below (highest precedence), keeping - # OBML -> Ossie -> OBML lossless. - ossie_datatype = field.get("datatype", "") - legacy_type = field.get("data_type", "") + # the stashed extension below, keeping OBML -> Ossie -> OBML lossless, + # unless the `datatype` was edited since. A non-string value (a + # hand-authored document) counts as absent rather than crashing. + ossie_datatype = field.get("datatype") + if not isinstance(ossie_datatype, str): + ossie_datatype = "" + legacy_type = field.get("data_type") + if not isinstance(legacy_type, str): + legacy_type = "" if ossie_datatype in OSSIE_DATATYPE_TO_OBML_ABSTRACT: abstract_type = OSSIE_DATATYPE_TO_OBML_ABSTRACT[ossie_datatype] elif legacy_type and legacy_type in OSSIE_TO_OBML_TYPE: @@ -421,10 +451,20 @@ def _convert_field(self, field: dict) -> tuple[str, dict]: try: ext_data = json.loads(ext.get("data", "{}")) # Restore the exact OBML abstractType stashed on export, so a - # narrowing datatype map (e.g. Decimal -> float) never - # degrades an OBML-origin round trip. - if ext_data.get("obml_abstract_type"): - col["abstractType"] = ext_data["obml_abstract_type"] + # narrowing datatype map (e.g. time_tz -> Time) never + # degrades an OBML-origin round trip. The stash yields to a + # `datatype` that no longer agrees with it: that is an edit + # made in Ossie after the export, and it is the newer fact. + stashed = ext_data.get("obml_abstract_type") + if isinstance(stashed, str) and stashed: + stashed_ossie = OBML_ABSTRACT_TO_OSSIE_DATATYPE.get(stashed) + edited = ( + ossie_datatype in OSSIE_DATATYPE_TO_OBML_ABSTRACT + and stashed_ossie is not None + and stashed_ossie != ossie_datatype + ) + if not edited: + col["abstractType"] = stashed if ext_data.get("obml_sql_type"): col["sqlType"] = ext_data["obml_sql_type"] if ext_data.get("obml_sql_precision") is not None: @@ -844,14 +884,20 @@ def _convert_metrics(self, ossie_metrics: list, ds_map: dict) -> tuple[dict, dic if target is not None: self._carry_foreign_extensions(m.get("custom_extensions"), target) # Ossie metric `datatype` -> OBML exact `dataType` (its natural - # home; `Decimal` -> decimal(p, s)). Don't override a dataType - # already restored from an OBML-origin extension, and skip - # Opaque/unknown (absent from the map). + # home; `Decimal` -> the model's decimal(p, s)). Opaque, unknown + # and non-string values have no mapping and change nothing. ossie_dt = m.get("datatype") - if ossie_dt and not target.get("dataType"): - obml_dt = OSSIE_DATATYPE_TO_OBML_PHYSICAL.get(ossie_dt) - if obml_dt: - target["dataType"] = obml_dt + obml_dt = ossie_metric_datatype_to_obml(ossie_dt, self._decimal_default) + if obml_dt is None: + continue + # A dataType restored from the OBML-origin stash is more exact + # than the map (`decimal(20, 6)`, `bigint`) and is kept while it + # still agrees with `datatype`. When it names a different type, + # `datatype` was edited in Ossie after the export, and the edit + # wins over the stale stash. + stashed_ossie = obml_datatype_to_ossie(target.get("dataType")) + if stashed_ossie is None or stashed_ossie != ossie_dt: + target["dataType"] = obml_dt return measures, metrics diff --git a/converters/orionbelt/tests/test_ossie_converter_datatype.py b/converters/orionbelt/tests/test_ossie_converter_datatype.py index 681d9ee7..752ec809 100644 --- a/converters/orionbelt/tests/test_ossie_converter_datatype.py +++ b/converters/orionbelt/tests/test_ossie_converter_datatype.py @@ -28,6 +28,7 @@ from typing import Any import ossie_orionbelt.converter as conv +from ossie_orionbelt._common import obml_datatype_to_ossie, obml_decimal_default def _ossie_field(name: str, **extra: Any) -> dict[str, Any]: @@ -236,3 +237,99 @@ def test_plain_measure_emits_no_datatype(self) -> None: ossie = conv.OBMLtoOssie(obml, model_name="s").convert() metric = ossie["semantic_model"][0]["metrics"][0] assert "datatype" not in metric + + +def _obml_with_measure(**measure: Any) -> dict[str, Any]: + return { + "dataObjects": { + "Orders": { + "code": "orders", + "columns": {"Amount": {"code": "amount", "abstractType": "float"}}, + } + }, + "measures": { + "Total": { + "columns": [{"dataObject": "Orders", "column": "Amount"}], + "resultType": "float", + "aggregation": "sum", + **measure, + } + }, + } + + +def _only_metric(ossie: dict[str, Any]) -> dict[str, Any]: + (metric,) = ossie["semantic_model"][0]["metrics"] + return metric + + +class TestMalformedDatatype: + """A hand-authored document with a non-string type must not abort conversion.""" + + def test_non_string_obml_data_type_emits_nothing(self) -> None: + assert obml_datatype_to_ossie(123) is None + assert obml_datatype_to_ossie(" ") is None + ossie = conv.OBMLtoOssie(_obml_with_measure(dataType=123), model_name="s").convert() + assert "datatype" not in _only_metric(ossie) + + def test_non_string_ossie_datatype_is_ignored(self) -> None: + # `price` is a heuristic float keyword, so the field falls back to it. + ossie = _ossie_model_with_metric("Decimal") + model = ossie["semantic_model"][0] + model["datasets"][0]["fields"] = [_ossie_field("price", datatype=["Integer"])] + model["metrics"][0]["datatype"] = 7 + model["metrics"][0]["expression"]["dialects"][0]["expression"] = "SUM(Orders.price)" + obml = conv.OssietoOBML(ossie).convert() + assert _obml_columns(obml)["price"]["abstractType"] == "float" + assert "dataType" not in obml["measures"]["Total"] + + +class TestDecimalDefaultFromSettings: + """``Decimal`` follows the model's ``settings.defaultNumericDataType``.""" + + def test_model_default_is_used(self) -> None: + obml = _obml_with_measure() + obml["settings"] = {"defaultNumericDataType": "decimal(20, 6)"} + ossie = conv.OBMLtoOssie(obml, model_name="s").convert() + _only_metric(ossie)["datatype"] = "Decimal" + back = conv.OssietoOBML(ossie).convert() + assert back["measures"]["Total"]["dataType"] == "decimal(20, 6)" + assert back["settings"] == {"defaultNumericDataType": "decimal(20, 6)"} + + def test_builtin_default_without_or_with_an_invalid_setting(self) -> None: + assert obml_decimal_default(None) == "decimal(18, 2)" + assert obml_decimal_default({"defaultNumericDataType": "bigint"}) == "decimal(18, 2)" + assert obml_decimal_default({"defaultNumericDataType": 5}) == "decimal(18, 2)" + obml = conv.OssietoOBML(_ossie_model_with_metric("Decimal")).convert() + assert obml["measures"]["Total"]["dataType"] == "decimal(18, 2)" + + +class TestEditedDatatypeBeatsStaleStash: + """An Ossie ``datatype`` edited after an OBML export wins over the stash.""" + + def test_metric_edit_wins(self) -> None: + ossie = conv.OBMLtoOssie(_obml_with_measure(dataType="integer"), model_name="s").convert() + assert _only_metric(ossie)["datatype"] == "Integer" + _only_metric(ossie)["datatype"] = "Float" + back = conv.OssietoOBML(ossie).convert() + assert back["measures"]["Total"]["dataType"] == "double" + + def test_metric_stash_that_still_agrees_stays_exact(self) -> None: + for data_type in ["decimal(20, 6)", "bigint"]: + obml = _obml_with_measure(dataType=data_type) + back = conv.OssietoOBML(conv.OBMLtoOssie(obml, model_name="s").convert()).convert() + assert back["measures"]["Total"]["dataType"] == data_type + + def test_field_edit_wins(self) -> None: + obml = { + "dataObjects": { + "Orders": { + "code": "orders", + "columns": {"Val": {"code": "val", "abstractType": "timestamp_tz"}}, + } + } + } + ossie = conv.OBMLtoOssie(obml, model_name="s").convert() + _ossie_fields(ossie)["val"]["datatype"] = "DateTime" + (col,) = _obml_columns(conv.OssietoOBML(ossie).convert()).values() + assert col["abstractType"] == "timestamp"