Skip to content

fix(honeydew): read and write flat semantic model documents #420

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
baruchoxman wants to merge 1 commit into
base: main
Choose a base branch
from
+41 −88
Open

fix(honeydew): read and write flat semantic model documents #420

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions converters/honeydew/README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ Honeydew documents this integration from its own side under

| Ossie concept | Honeydew concept |
|-------------|-----------------|
| `semantic_model.name` | `workspace.yml name` |
| `name` (document root) | `workspace.yml name` |
| `dataset` | Entity + dataset files under `schema/<entity>/` |
| `dataset.source` | `dataset.sql` |
| `dataset.primary_key` | `entity.keys` |
Expand All @@ -50,7 +50,7 @@ Honeydew documents this integration from its own side under

| Honeydew concept | Ossie concept |
|-----------------|-------------|
| `workspace.name` | `semantic_model.name` |
| `workspace.name` | `name` (document root) |
| Entity + primary dataset | `dataset` |
| `entity.keys` | `dataset.primary_key` (and `dataset.unique_keys`) |
| `dataset.attributes` (columns) | `fields` with `ANSI_SQL` expression = column name |
Expand Down
39 changes: 12 additions & 27 deletions converters/honeydew/src/ossie_honeydew/converter.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@
_OSSIE_METADATA_SECTION = "ossie"
# Workspaces written before the Ossie rebrand named the section "osi". Still
# read it, so exporting such a workspace does not silently drop the fields it
# preserves (ai_context, label, unique_keys, custom_extensions, vendors).
# preserves (ai_context, label, unique_keys, custom_extensions).
_LEGACY_OSSIE_METADATA_SECTION = "osi"
_HD_ATTR_KEYS = ("display_name", "hidden", "folder", "format_string", "timegrain")

Expand Down Expand Up @@ -98,38 +98,33 @@ def convert_ossie_to_honeydew(ossie_yaml_str: str) -> dict[str, str]:
f"Unsupported Ossie version '{version_str}'. Supported: {SUPPORTED_OSSIE_VERSION}"
)

semantic_models = root.get("semantic_model")
if not isinstance(semantic_models, list) or not semantic_models:
raise HoneydewConversionError("'semantic_model' must be a non-empty list")

if len(semantic_models) > 1:
warnings.warn(
f"Ossie YAML contains {len(semantic_models)} semantic models; "
"only the first will be converted"
if "semantic_model" in root:
raise HoneydewConversionError(
"Document uses the legacy 'semantic_model:' wrapper. An Ossie document now "
"defines exactly one semantic model at its root; move the model's properties "
"up to the document root alongside 'version'."
)

vendors = [v for v in (root.get("vendors") or []) if v != HONEYDEW_VENDOR]
return _model_to_files(semantic_models[0], extra_vendors=vendors)
return _model_to_files(root)


def _model_to_files(sm: dict[str, Any], *, extra_vendors: list[str] | None = None) -> dict[str, str]:
def _model_to_files(sm: dict[str, Any]) -> dict[str, str]:
name = sm.get("name")
if not name:
raise HoneydewConversionError("Missing 'name' in semantic model")
raise HoneydewConversionError("Missing 'name' in Ossie document")

files: dict[str, str] = {}

workspace: dict[str, Any] = {"type": "workspace", "name": name}
if sm.get("description"):
workspace["description"] = sm["description"]

# Preserve model-level ai_context, non-HONEYDEW custom_extensions, and extra vendors
# Preserve model-level ai_context and non-HONEYDEW custom_extensions
model_ai_ctx = sm.get("ai_context")
model_ext = [e for e in (sm.get("custom_extensions") or []) if e.get("vendor_name") != HONEYDEW_VENDOR]
ws_meta = _build_ossie_metadata(
ai_context=model_ai_ctx,
custom_extensions=model_ext or None,
extra_vendors=extra_vendors or None,
)
if ws_meta:
workspace["metadata"] = [ws_meta]
Expand Down Expand Up @@ -606,14 +601,7 @@ def convert_honeydew_to_ossie(workspace_dir: str) -> str:
if ossie_metrics:
sm["metrics"] = ossie_metrics

extra_vendors = ws_ossie_meta.get("vendors") or []
vendors = [HONEYDEW_VENDOR] + [v for v in extra_vendors if v != HONEYDEW_VENDOR]
root: dict[str, Any] = {
"version": SUPPORTED_OSSIE_VERSION,
"vendors": vendors,
"semantic_model": [sm],
}
return _dump(root)
return _dump({"version": SUPPORTED_OSSIE_VERSION, **sm})


def _read_entity_dir(entity_dir: str, entity_name: str) -> dict[str, Any]:
Expand Down Expand Up @@ -934,7 +922,6 @@ def _build_ossie_metadata(
label: str | None = None,
unique_keys: Any = None,
custom_extensions: list | None = None,
extra_vendors: list[str] | None = None,
) -> dict[str, Any] | None:
"""Build a Honeydew metadata entry that stores Ossie-only fields for round-tripping."""
items: list[dict[str, Any]] = []
Expand All @@ -948,8 +935,6 @@ def _build_ossie_metadata(
items.append({"name": "unique_keys", "value": json.dumps(unique_keys)})
if custom_extensions:
items.append({"name": "custom_extensions", "value": json.dumps(custom_extensions)})
if extra_vendors:
items.append({"name": "vendors", "value": json.dumps(extra_vendors)})

if not items:
return None
Expand All @@ -975,7 +960,7 @@ def _read_ossie_metadata(obj: dict[str, Any]) -> dict[str, Any]:
result[key] = raw
elif key == "label":
result[key] = raw
elif key in ("unique_keys", "custom_extensions", "vendors"):
elif key in ("unique_keys", "custom_extensions"):
try:
result[key] = json.loads(raw)
except (json.JSONDecodeError, TypeError):
Expand Down
86 changes: 27 additions & 59 deletions converters/honeydew/tests/test_ossie_honeydew_converter.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@

def _ossie(model_dict):
return yaml.dump(
{"version": OSSIE_VERSION, "semantic_model": [model_dict]},
{"version": OSSIE_VERSION, **model_dict},
default_flow_style=False,
sort_keys=False,
)
Expand Down Expand Up @@ -134,13 +134,15 @@ def _write_workspace(tmp_dir, workspace_name, entities):


def _ossie_roundtrip(model_dict, tmp_path):
"""Ossie → Honeydew → Ossie; returns the semantic model dict."""
"""Ossie → Honeydew → Ossie; returns the round-tripped model without 'version'."""
files = convert_ossie_to_honeydew(_ossie(model_dict))
for rel_path, content in files.items():
p = tmp_path / rel_path
p.parent.mkdir(parents=True, exist_ok=True)
p.write_text(content)
return yaml.safe_load(convert_honeydew_to_ossie(str(tmp_path)))["semantic_model"][0]
doc = yaml.safe_load(convert_honeydew_to_ossie(str(tmp_path)))
assert doc.pop("version") == OSSIE_VERSION
return doc


def _honeydew_roundtrip(entities, tmp_path):
Expand Down Expand Up @@ -596,32 +598,27 @@ def test_ossie_to_honeydew_metric_entity_hint_overrides_expression():

def test_ossie_to_honeydew_invalid_version_raises():
with pytest.raises(HoneydewConversionError, match="Unsupported"):
convert_ossie_to_honeydew("version: '9.9.9'\nsemantic_model:\n - name: m\n")
convert_ossie_to_honeydew("version: '9.9.9'\nname: m\ndatasets: []\n")


def test_ossie_to_honeydew_missing_semantic_model_raises():
with pytest.raises(HoneydewConversionError):
convert_ossie_to_honeydew(f"version: '{OSSIE_VERSION}'\n")
def test_ossie_to_honeydew_missing_name_raises():
with pytest.raises(HoneydewConversionError, match="Missing 'name'"):
convert_ossie_to_honeydew(f"version: '{OSSIE_VERSION}'\ndatasets: []\n")


def test_ossie_to_honeydew_multiple_models_warns():
doc = yaml.dump({"version": OSSIE_VERSION, "semantic_model": [
{"name": "m1", "datasets": []},
{"name": "m2", "datasets": []},
]})
with warnings.catch_warnings(record=True) as w:
warnings.simplefilter("always")
files = convert_ossie_to_honeydew(doc)
assert any("only the first" in str(x.message) for x in w)
assert yaml.safe_load(files["workspace.yml"]) == {"type": "workspace", "name": "m1"}
def test_ossie_to_honeydew_legacy_wrapper_raises():
"""The pre-#383 'semantic_model:' wrapper is no longer a valid document."""
doc = yaml.dump({"version": OSSIE_VERSION, "semantic_model": [{"name": "m", "datasets": []}]})
with pytest.raises(HoneydewConversionError, match="legacy 'semantic_model:' wrapper"):
convert_ossie_to_honeydew(doc)


# ─────────────────────────────────────────────────────────────────────────────
# Honeydew → Ossie: full document
# ─────────────────────────────────────────────────────────────────────────────

def _hd_root(sm):
return {"version": OSSIE_VERSION, "vendors": ["HONEYDEW"], "semantic_model": [sm]}
return {"version": OSSIE_VERSION, **sm}


def _ansi(expr):
Expand Down Expand Up @@ -813,8 +810,7 @@ def test_honeydew_to_ossie_missing_workspace_raises(tmp_path):
def test_honeydew_to_ossie_missing_schema_dir_empty_model(tmp_path):
(tmp_path / "workspace.yml").write_text(yaml.dump({"type": "workspace", "name": "ws"}))
result = yaml.safe_load(convert_honeydew_to_ossie(str(tmp_path)))
assert result == {"version": OSSIE_VERSION, "vendors": ["HONEYDEW"],
"semantic_model": [{"name": "ws", "datasets": []}]}
assert result == {"version": OSSIE_VERSION, "name": "ws", "datasets": []}


def test_honeydew_to_ossie_empty_metric_sql_skipped(tmp_path):
Expand All @@ -824,7 +820,7 @@ def test_honeydew_to_ossie_empty_metric_sql_skipped(tmp_path):
"datatype": "number", "sql": ""}]}])
with warnings.catch_warnings(record=True):
result = yaml.safe_load(convert_honeydew_to_ossie(str(tmp_path)))
assert "metrics" not in result["semantic_model"][0]
assert "metrics" not in result


def test_honeydew_to_ossie_duplicate_relations_deduplicated(tmp_path):
Expand All @@ -839,7 +835,7 @@ def test_honeydew_to_ossie_duplicate_relations_deduplicated(tmp_path):
"dataset_attrs": []},
])
result = yaml.safe_load(convert_honeydew_to_ossie(str(tmp_path)))
assert len(result["semantic_model"][0].get("relationships", [])) == 1
assert len(result.get("relationships", [])) == 1


def test_honeydew_to_ossie_relation_target_columns_are_unique_keys(tmp_path):
Expand All @@ -855,7 +851,7 @@ def test_honeydew_to_ossie_relation_target_columns_are_unique_keys(tmp_path):
{"name": "customers", "keys": ["id"], "key_dataset": "customers",
"sql": "db.s.customers", "dataset_attrs": []},
])
sm = yaml.safe_load(convert_honeydew_to_ossie(str(tmp_path)))["semantic_model"][0]
sm = yaml.safe_load(convert_honeydew_to_ossie(str(tmp_path)))
datasets = {ds["name"]: ds for ds in sm["datasets"]}
rel = sm["relationships"][0]
target_ds = datasets[rel["to"]]
Expand Down Expand Up @@ -1002,8 +998,7 @@ def test_ossie_roundtrip_tpcds_example(tmp_path):
p = tmp_path / rel_path
p.parent.mkdir(parents=True, exist_ok=True)
p.write_text(content)
result = yaml.safe_load(convert_honeydew_to_ossie(str(tmp_path)))
sm = result["semantic_model"][0]
sm = yaml.safe_load(convert_honeydew_to_ossie(str(tmp_path)))
assert sm["name"] == "tpcds_retail_model"
ds_names = {ds["name"] for ds in sm["datasets"]}
assert "store_sales" in ds_names and "customer" in ds_names
Expand Down Expand Up @@ -1417,31 +1412,6 @@ def test_connectionless_relation_warns():
}


# ─────────────────────────────────────────────────────────────────────────────
# Vendors round-trip
# ─────────────────────────────────────────────────────────────────────────────

@pytest.mark.parametrize("input_vendors,expected_vendors", [
(["SNOWFLAKE", "HONEYDEW"], ["HONEYDEW", "SNOWFLAKE"]),
(["SNOWFLAKE"], ["HONEYDEW", "SNOWFLAKE"]),
(["HONEYDEW"], ["HONEYDEW"]),
])
def test_vendors_roundtrip(tmp_path, input_vendors, expected_vendors):
doc = yaml.dump({
"version": OSSIE_VERSION,
"vendors": input_vendors,
"semantic_model": [{"name": "m", "datasets": []}],
})
files = convert_ossie_to_honeydew(doc)
for rel_path, content in files.items():
p = tmp_path / rel_path
p.parent.mkdir(parents=True, exist_ok=True)
p.write_text(content)
result = yaml.safe_load(convert_honeydew_to_ossie(str(tmp_path)))
assert result == {"version": OSSIE_VERSION, "vendors": expected_vendors,
"semantic_model": [{"name": "m", "datasets": []}]}


# ─────────────────────────────────────────────────────────────────────────────
# main() CLI smoke tests
# ─────────────────────────────────────────────────────────────────────────────
Expand All @@ -1451,9 +1421,8 @@ def test_main_ossie_to_honeydew(tmp_path):
input_file = tmp_path / "model.yaml"
input_file.write_text(yaml.dump({
"version": OSSIE_VERSION,
"semantic_model": [{"name": "m", "datasets": [
{"name": "orders", "source": "db.s.orders", "fields": []}
]}],
"name": "m",
"datasets": [{"name": "orders", "source": "db.s.orders", "fields": []}],
}))
output_dir = tmp_path / "out"
result = subprocess.run(
Expand Down Expand Up @@ -1482,21 +1451,20 @@ def test_main_honeydew_to_ossie(tmp_path):
assert result.returncode == 0
assert yaml.safe_load(output_file.read_text()) == {
"version": OSSIE_VERSION,
"vendors": ["HONEYDEW"],
"semantic_model": [{"name": "ws", "datasets": [
"name": "ws",
"datasets": [
{"name": "orders", "source": "DB.S.ORDERS", "primary_key": ["id"],
"unique_keys": [["id"]]},
]}],
],
}


def test_main_path_traversal_rejected(tmp_path):
import subprocess
input_file = tmp_path / "model.yaml"
input_file.write_text(
f"version: '{OSSIE_VERSION}'\nsemantic_model:\n"
" - name: m\n datasets:\n"
" - name: '../../evil'\n source: db.s.evil\n fields: []\n"
f"version: '{OSSIE_VERSION}'\nname: m\ndatasets:\n"
" - name: '../../evil'\n source: db.s.evil\n fields: []\n"
)
output_dir = tmp_path / "out"
result = subprocess.run(
Expand Down
Loading