diff --git a/faircode/loaders_extra.py b/faircode/loaders_extra.py index 2ff8d20..d1ec1de 100644 --- a/faircode/loaders_extra.py +++ b/faircode/loaders_extra.py @@ -36,6 +36,7 @@ def read_table(path: str) -> pd.DataFrame: if suffix == ".json": with open(path, "r", encoding="utf-8") as f: raw = f.read() + try: parsed = json.loads(raw) except json.JSONDecodeError as exc: @@ -43,15 +44,63 @@ def read_table(path: str) -> pd.DataFrame: # file) is an internal/version-specific message. Fail fast with # a clear message instead, mirroring the JS engine's parseJSON(). raise ValueError(f"Unsupported JSON format (not valid JSON: {exc}).") from exc - # Detect split-orient ({"columns": [...], "data": [...]}, optionally - # "index") from the parsed shape rather than trying the default - # (records) orientation first and catching its ValueError: a split - # file that omits the optional "index" key parses without error - # under the default orientation too - as two columns literally named - # "columns" and "data" - so the ValueError this used to rely on - # never fires, and the wrong shape comes back silently. + + # Detect split-orient JSON: + # {"columns": [...], "data": [...]}, optionally with "index". + # + # This must be checked before the generic dict-of-dicts detection + # below because columns-oriented JSON is also represented as a + # dictionary of dictionaries. if isinstance(parsed, dict) and {"columns", "data"} <= parsed.keys(): return pd.read_json(path, orient="split") + + # Detect index-oriented JSON. + # + # pandas' "columns" and "index" orientations both use a + # dict-of-dicts representation, so their shapes are inherently + # ambiguous in JSON. + # + # Heuristic: when the dict-of-dicts is scalar and rectangular, prefer + # the orientation implied by which dimension is larger (rows vs + # columns). For square cases, use value type-homogeneity to prefer + # columns-orient (columns are often homogeneous; rows often mix types). + if isinstance(parsed, dict) and parsed and all( + isinstance(value, dict) for value in parsed.values() + ): + inner_key_sets = [set(row.keys()) for row in parsed.values()] + + same_inner_keys = len({frozenset(keys) for keys in inner_key_sets}) == 1 + + if same_inner_keys: + inner_keys = inner_key_sets[0] + + cells_are_scalar = all( + not isinstance(cell, (dict, list)) + for row in parsed.values() + for cell in row.values() + ) + + if cells_are_scalar: + outer_n = len(parsed) + inner_n = len(inner_keys) + + if outer_n != inner_n: + orient = "index" if outer_n > inner_n else "columns" + return pd.read_json(path, orient=orient) + + # Square case: prefer columns-orient when values are more type-homogeneous + # by column than by row. + row_type_score = sum( + len({type(cell) for cell in row.values()}) + for row in parsed.values() + ) + col_type_score = sum( + len({type(parsed[row_key][col_key]) for row_key in parsed}) + for col_key in inner_keys + ) + + orient = "index" if row_type_score > col_type_score else "columns" + return pd.read_json(path, orient=orient) return pd.read_json(path) if suffix == ".parquet": @@ -84,4 +133,5 @@ def get_xlsx_sheet_info(path: str) -> tuple[str, list[str]] | None: return None if not book.sheet_names: return None - return book.sheet_names[0], book.sheet_names[1:] + + return book.sheet_names[0], book.sheet_names[1:] \ No newline at end of file diff --git a/tests/test_json_edge_cases.py b/tests/test_json_edge_cases.py index de63658..a6dee72 100644 --- a/tests/test_json_edge_cases.py +++ b/tests/test_json_edge_cases.py @@ -9,6 +9,7 @@ import subprocess from pathlib import Path +import pandas as pd import pytest from faircode.loaders_extra import read_table @@ -101,3 +102,23 @@ def test_deeply_nested_json_python_current_behavior(tmp_path): path = _write_json(tmp_path, json.dumps({"a": {"b": {"c": 1}}})) df = read_table(str(path)) assert df.to_dict(orient="records") == [{"a": {"c": 1}}] + + +# ── Index-oriented JSON ────────────────────────────────────────────────────── +def test_index_oriented_json_preserves_dataframe_shape(tmp_path): + original = pd.DataFrame( + {"sex": ["M", "F"], "age": [30, 40]}, + index=["a", "b"], + ) + + index_path = tmp_path / "index.json" + original.to_json(index_path, orient="index") + loaded_index = read_table(str(index_path)) + pd.testing.assert_frame_equal(loaded_index, original) + + # Ensure columns-oriented dict-of-dicts with a non-default index isn't + # misclassified as index-oriented and transposed. + columns_path = tmp_path / "columns.json" + original.to_json(columns_path, orient="columns") + loaded_columns = read_table(str(columns_path)) + pd.testing.assert_frame_equal(loaded_columns, original) \ No newline at end of file