Skip to content

Commit e00ca8b

Browse files
LukasGoldLukasGold
committed
fix: handle empty SMW ask results in semantic_search
- SMW serialises an empty ask result set as a JSON array, not an object, so any zero-result query raised AttributeError instead of returning [] - normalise the payload once via _ask_results_as_dict() - warn when a query hits its limit, which silently truncated results - warn when entries are dropped by the exists != "1" filter - refs #145, #111
1 parent 0d32bc2 commit e00ca8b

2 files changed

Lines changed: 177 additions & 7 deletions

File tree

‎src/osw/wiki_tools.py‎

Lines changed: 48 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
import getpass
2+
import warnings
23
from typing import Dict, List, Optional, Tuple, Union
34

45
import mwclient
@@ -216,6 +217,33 @@ def prefix_search_(single_text) -> Union[List[str], dict]:
216217
# return page_list # original return
217218

218219

220+
def _ask_results_as_dict(results: Union[dict, list]) -> dict:
221+
"""Normalise the ``results`` payload of an SMW ``ask`` response to a mapping.
222+
223+
SMW serialises a non-empty result set as a JSON object keyed by page title,
224+
but an empty one as a JSON array, which would otherwise break attribute
225+
access on the result.
226+
227+
Parameters
228+
----------
229+
results :
230+
The ``result["query"]["results"]`` payload of an SMW ``ask`` response.
231+
232+
Returns
233+
-------
234+
result:
235+
The payload normalised to a dict, keyed by page title (or index if a
236+
title is unavailable).
237+
"""
238+
if isinstance(results, dict):
239+
return results
240+
if not results:
241+
return {}
242+
return {
243+
page.get("fulltext", str(index)): page for index, page in enumerate(results)
244+
}
245+
246+
219247
def semantic_search(
220248
site: mwclient.client.Site, query: Union[str, List[str], SearchParam]
221249
) -> Union[List[str], List[dict]]:
@@ -242,26 +270,39 @@ def semantic_search_(single_query):
242270
page_list = list()
243271
single_query += f"|limit={query.limit}"
244272
result = site.api("ask", query=single_query, format="json")
273+
results = _ask_results_as_dict(result["query"]["results"])
274+
n = len(results)
245275
if query.debug:
246-
if len(result["query"]["results"]) == 0:
276+
if n == 0:
247277
print(f"Query '{single_query}' returned no results")
248278
else:
249-
print(
250-
"Query '{}' returned {} results".format(
251-
single_query, len(result["query"]["results"])
252-
)
253-
)
279+
print(f"Query '{single_query}' returned {n} results")
280+
if n >= query.limit:
281+
warnings.warn(
282+
f"Query '{single_query}' returned {n} results, which meets the "
283+
f"requested limit of {query.limit}. Results are truncated - raise "
284+
f"the limit or page through with '|offset=' to retrieve the "
285+
f"remainder."
286+
)
254287
if query.return_json:
255288
return result
256289

257-
for page in result["query"]["results"].values():
290+
dropped = 0
291+
for page in results.values():
258292
title = page["fulltext"]
259293
exists = page["exists"]
260294
if "#" not in title and query.debug:
261295
print(title)
262296
# original position of "page_list.append(title)" line
263297
if exists == "1":
264298
page_list.append(title)
299+
else:
300+
dropped += 1
301+
if dropped > 0:
302+
warnings.warn(
303+
f"Query '{single_query}': {dropped} of {n} results were dropped "
304+
f"because the wiki reported them as non-existing pages."
305+
)
265306
return page_list
266307

267308
if query.parallel:

‎tests/test_wiki_tools.py‎

Lines changed: 129 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,8 @@
1+
import warnings
12
from unittest.mock import MagicMock
23

4+
import pytest
5+
36
import osw.wiki_tools as wt
47

58

@@ -29,6 +32,12 @@ def _ask_result(*titles):
2932
}
3033

3134

35+
def _ask_result_empty():
36+
"""Build the SMW ``ask`` API result dict for a zero-result query, mirroring
37+
SMW's behaviour of serialising an empty result set as a JSON array."""
38+
return {"query": {"results": []}}
39+
40+
3241
def test_semantic_search_return_json_single_query_returns_list_with_dict():
3342
result = _ask_result("Item:OSW1")
3443
site = MagicMock()
@@ -78,6 +87,126 @@ def test_semantic_search_returns_flat_list_of_titles():
7887
assert out == ["Item:OSW1", "Item:OSW2"]
7988

8089

90+
def test_semantic_search_zero_results_returns_empty_list():
91+
site = MagicMock()
92+
site.api.return_value = _ask_result_empty()
93+
94+
out = wt.semantic_search(site, "[[HasType::Category:Nonexistent]]")
95+
96+
assert out == []
97+
98+
99+
def test_semantic_search_zero_results_return_json_returns_list_with_dict():
100+
result = _ask_result_empty()
101+
site = MagicMock()
102+
site.api.return_value = result
103+
104+
out = wt.semantic_search(
105+
site,
106+
wt.SearchParam(query="[[HasType::Category:Nonexistent]]", return_json=True),
107+
)
108+
109+
assert out == [result]
110+
111+
112+
def test_semantic_search_batch_with_one_zero_result_query():
113+
result_a = _ask_result("Item:OSW1")
114+
result_b = _ask_result_empty()
115+
result_c = _ask_result("Item:OSW3")
116+
site = MagicMock()
117+
site.api.side_effect = [result_a, result_b, result_c]
118+
119+
out = wt.semantic_search(
120+
site,
121+
wt.SearchParam(
122+
query=[
123+
"[[HasType::Category:A]]",
124+
"[[HasType::Category:B]]",
125+
"[[HasType::Category:C]]",
126+
]
127+
),
128+
)
129+
130+
assert out == ["Item:OSW1", "Item:OSW3"]
131+
132+
133+
def test_semantic_search_parallel_batch_with_one_zero_result_query():
134+
results = [
135+
_ask_result("Item:OSW1"),
136+
_ask_result("Item:OSW2"),
137+
_ask_result_empty(),
138+
_ask_result("Item:OSW4"),
139+
_ask_result("Item:OSW5"),
140+
_ask_result("Item:OSW6"),
141+
]
142+
site = MagicMock()
143+
site.api.side_effect = results
144+
145+
out = wt.semantic_search(
146+
site,
147+
wt.SearchParam(
148+
query=[
149+
"[[HasType::Category:A]]",
150+
"[[HasType::Category:B]]",
151+
"[[HasType::Category:C]]",
152+
"[[HasType::Category:D]]",
153+
"[[HasType::Category:E]]",
154+
"[[HasType::Category:F]]",
155+
]
156+
),
157+
)
158+
159+
assert sorted(out) == [
160+
"Item:OSW1",
161+
"Item:OSW2",
162+
"Item:OSW4",
163+
"Item:OSW5",
164+
"Item:OSW6",
165+
]
166+
167+
168+
def test_semantic_search_truncation_warning():
169+
titles = [f"Item:OSW{i}" for i in range(5)]
170+
result = _ask_result(*titles)
171+
site = MagicMock()
172+
site.api.return_value = result
173+
174+
with pytest.warns(UserWarning, match="truncated"):
175+
out = wt.semantic_search(
176+
site, wt.SearchParam(query="[[HasType::Category:Item]]", limit=5)
177+
)
178+
179+
assert sorted(out) == sorted(titles)
180+
181+
182+
def test_semantic_search_no_truncation_warning_below_limit():
183+
titles = [f"Item:OSW{i}" for i in range(5)]
184+
result = _ask_result(*titles)
185+
site = MagicMock()
186+
site.api.return_value = result
187+
188+
with warnings.catch_warnings(record=True) as caught:
189+
warnings.simplefilter("always")
190+
out = wt.semantic_search(
191+
site, wt.SearchParam(query="[[HasType::Category:Item]]", limit=1000)
192+
)
193+
194+
assert not any("truncated" in str(w.message) for w in caught)
195+
assert sorted(out) == sorted(titles)
196+
197+
198+
def test_semantic_search_exists_drop_warning():
199+
result = _ask_result("Item:OSW1", "Item:OSW2")
200+
result["query"]["results"]["Item:OSW2"]["exists"] = ""
201+
site = MagicMock()
202+
site.api.return_value = result
203+
204+
with pytest.warns(UserWarning, match="non-existing"):
205+
out = wt.semantic_search(site, "[[HasType::Category:Item]]")
206+
207+
assert out == ["Item:OSW1"]
208+
209+
81210
def _prefixsearch_result(*titles):
82211
"""Build a minimal MediaWiki ``prefixsearch`` API result dict."""
83212
return {

0 commit comments

Comments
 (0)