Skip to content

Commit 6c672dc

Browse files
committed
fix(core): write a stub for a $ref whose page does not exist
- write an empty schema so the rewritten $ref still resolves on disk - carry the recursive fetch's error and warning messages up to the caller - merge warning_messages in fetch_schema, which only merged errors - add get_model_dir_path so both writers agree on the location
1 parent 0d32bc2 commit 6c672dc

2 files changed

Lines changed: 146 additions & 5 deletions

File tree

‎src/osw/core.py‎

Lines changed: 63 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,46 @@
7171
_logger = logging.getLogger(__name__)
7272

7373

74+
def get_model_dir_path() -> str:
75+
"""The directory the fetched json schemas are written to"""
76+
return os.path.join(os.path.dirname(os.path.abspath(__file__)), "model")
77+
78+
79+
def write_schema_stub(model_dir_path: str, schema_name: str) -> str:
80+
"""Writes an empty schema, so a $ref pointing at it still resolves
81+
82+
A $ref is rewritten to a local file name before the page it names is
83+
fetched. When that page turns out not to exist, nothing writes the file,
84+
and datamodel-code-generator later fails with FileNotFoundError far away
85+
from the cause. An empty schema keeps generation going, and the reason is
86+
reported in the FetchSchemaResult instead.
87+
"""
88+
schema_path = os.path.join(model_dir_path, schema_name + ".json")
89+
os.makedirs(os.path.dirname(schema_path), exist_ok=True)
90+
with open(schema_path, "w", encoding="utf-8") as f:
91+
f.write("{}")
92+
return schema_path
93+
94+
95+
def collect_messages(
96+
target: Optional[List[str]], source: Optional[List[str]]
97+
) -> Optional[List[str]]:
98+
"""Adds the messages in source to target, skipping duplicates
99+
100+
_FetchSchemaParam is copied shallowly for a recursive fetch, so target and
101+
source can be the very same list. That case is already merged and is left
102+
alone rather than iterated while being appended to.
103+
"""
104+
if not source or target is source:
105+
return target
106+
if target is None:
107+
target = []
108+
for message in source:
109+
if message not in target:
110+
target.append(message)
111+
return target
112+
113+
74114
# Reusable type definitions
75115
class OverwriteOptions(Enum):
76116
"""Options for overwriting properties"""
@@ -477,13 +517,15 @@ def fetch_schema(
477517

478518
# merge unique results and return
479519
merged_result = OSW.FetchSchemaResult(
480-
fetched_schema_titles=[], error_messages=[]
520+
fetched_schema_titles=[], error_messages=[], warning_messages=[]
481521
)
482522
for result in results:
483523
if result.fetched_schema_titles:
484524
merged_result.fetched_schema_titles.extend(result.fetched_schema_titles)
485525
if result.error_messages:
486526
merged_result.error_messages.extend(result.error_messages)
527+
if result.warning_messages:
528+
merged_result.warning_messages.extend(result.warning_messages)
487529
return OSW.FetchSchemaResult(
488530
fetched_schema_titles=(
489531
list(set(merged_result.fetched_schema_titles))
@@ -495,6 +537,11 @@ def fetch_schema(
495537
if len(merged_result.error_messages) > 0
496538
else None
497539
),
540+
warning_messages=(
541+
list(set(merged_result.warning_messages))
542+
if len(merged_result.warning_messages) > 0
543+
else None
544+
),
498545
)
499546

500547
class _FetchSchemaParam(BaseModel):
@@ -569,6 +616,8 @@ def _fetch_schema(
569616
]
570617
if not page.exists:
571618
print(f"Error: Page {schema_title} does not exist")
619+
# the $ref that led here was already rewritten to this file name
620+
write_schema_stub(get_model_dir_path(), schema_name)
572621
return OSW.FetchSchemaResult(
573622
fetched_schema_titles=fetchSchemaParam.fetched_schema_titles,
574623
warning_messages=fetchSchemaParam.warning_messages,
@@ -606,6 +655,7 @@ def _fetch_schema(
606655
schema = json.loads(schema_str.replace("$ref", "dollarref"))
607656

608657
jsonpath_expr = parse("$..dollarref")
658+
ref_error_messages = None
609659
for match in jsonpath_expr.find(schema):
610660
# value = "https://" + self.mw_site.host + match.value
611661
if match.value.startswith("#"):
@@ -628,11 +678,18 @@ def _fetch_schema(
628678
_param = fetchSchemaParam.copy()
629679
_param.root = False
630680
_param.schema_title = ref_schema_title
631-
self._fetch_schema(_param) # resolve references recursive
681+
ref_result = self._fetch_schema(_param) # resolve refs recursive
682+
# the recursive call is the only place that knows why a
683+
# referenced schema could not be fetched, so its messages have
684+
# to be carried up rather than dropped
685+
ref_error_messages = collect_messages(
686+
ref_error_messages, ref_result.error_messages
687+
)
688+
fetchSchemaParam.warning_messages = collect_messages(
689+
fetchSchemaParam.warning_messages, ref_result.warning_messages
690+
)
632691

633-
model_dir_path = os.path.join(
634-
os.path.dirname(os.path.abspath(__file__)), "model"
635-
) # src/model
692+
model_dir_path = get_model_dir_path() # src/model
636693
schema_path = os.path.join(model_dir_path, schema_name + ".json")
637694
os.makedirs(os.path.dirname(schema_path), exist_ok=True)
638695
with open(schema_path, "w", encoding="utf-8") as f:
@@ -1005,6 +1062,7 @@ def _fetch_schema(
10051062
return OSW.FetchSchemaResult(
10061063
fetched_schema_titles=fetchSchemaParam.fetched_schema_titles,
10071064
warning_messages=fetchSchemaParam.warning_messages,
1065+
error_messages=ref_error_messages,
10081066
)
10091067

10101068
def install_dependencies(

‎tests/test_schema_ref_stub.py‎

Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,83 @@
1+
"""Unit tests for the missing-schema-page handling in osw.core.
2+
3+
Regression guard for #134: a $ref is rewritten to a local file name before the
4+
page it names is fetched. When that page does not exist, no file was written
5+
and no message reached the caller, so the failure surfaced much later as a
6+
FileNotFoundError inside datamodel-code-generator.
7+
"""
8+
9+
import json
10+
import os
11+
12+
import pytest
13+
14+
from osw.core import collect_messages, get_model_dir_path, write_schema_stub
15+
16+
17+
def test_stub_is_valid_empty_json(tmp_path):
18+
path = write_schema_stub(str(tmp_path), "undefined")
19+
20+
with open(path, encoding="utf-8") as f:
21+
assert json.load(f) == {}
22+
23+
24+
def test_stub_lands_where_the_rewritten_ref_points(tmp_path):
25+
"""The $ref becomes '<name>.json' relative to the model dir."""
26+
path = write_schema_stub(str(tmp_path), "undefined")
27+
28+
assert os.path.basename(path) == "undefined.json"
29+
assert os.path.dirname(path) == str(tmp_path)
30+
31+
32+
def test_stub_creates_missing_directories(tmp_path):
33+
target = tmp_path / "nested" / "deeper"
34+
35+
path = write_schema_stub(str(target), "undefined")
36+
37+
assert os.path.isfile(path)
38+
39+
40+
def test_stub_overwrites_an_existing_file(tmp_path):
41+
(tmp_path / "undefined.json").write_text('{"stale": true}', encoding="utf-8")
42+
43+
path = write_schema_stub(str(tmp_path), "undefined")
44+
45+
with open(path, encoding="utf-8") as f:
46+
assert json.load(f) == {}
47+
48+
49+
def test_model_dir_is_next_to_core():
50+
model_dir = get_model_dir_path()
51+
52+
assert os.path.basename(model_dir) == "model"
53+
assert os.path.isfile(os.path.join(model_dir, "entity.py"))
54+
55+
56+
@pytest.mark.parametrize("source", [None, []])
57+
def test_nothing_to_collect_leaves_the_target_alone(source):
58+
assert collect_messages(None, source) is None
59+
assert collect_messages(["a"], source) == ["a"]
60+
61+
62+
def test_messages_are_collected_into_an_absent_target():
63+
assert collect_messages(None, ["a"]) == ["a"]
64+
65+
66+
def test_messages_are_appended_without_duplicates():
67+
assert collect_messages(["a"], ["a", "b"]) == ["a", "b"]
68+
69+
70+
def test_a_shared_list_is_not_iterated_while_appended_to():
71+
"""_FetchSchemaParam.copy() is shallow, so both sides can be one list."""
72+
shared = ["a"]
73+
74+
assert collect_messages(shared, shared) is shared
75+
assert shared == ["a"]
76+
77+
78+
def test_collecting_does_not_mutate_the_source():
79+
source = ["b"]
80+
81+
collect_messages(["a"], source)
82+
83+
assert source == ["b"]

0 commit comments

Comments
 (0)