Fix index-oriented JSON parsing

faircode/loaders_extra.py

@@ -36,22 +36,71 @@ def read_table(path: str) -> pd.DataFrame:
     if suffix == ".json":
         with open(path, "r", encoding="utf-8") as f:
             raw = f.read()
+
         try:
             parsed = json.loads(raw)
         except json.JSONDecodeError as exc:
             # pandas' own parser error for malformed JSON (e.g. a truncated
             # file) is an internal/version-specific message. Fail fast with
             # a clear message instead, mirroring the JS engine's parseJSON().
             raise ValueError(f"Unsupported JSON format (not valid JSON: {exc}).") from exc
-        # Detect split-orient ({"columns": [...], "data": [...]}, optionally
-        # "index") from the parsed shape rather than trying the default
-        # (records) orientation first and catching its ValueError: a split
-        # file that omits the optional "index" key parses without error
-        # under the default orientation too - as two columns literally named
-        # "columns" and "data" - so the ValueError this used to rely on
-        # never fires, and the wrong shape comes back silently.
+
+        # Detect split-orient JSON:
+        # {"columns": [...], "data": [...]}, optionally with "index".
+        #
+        # This must be checked before the generic dict-of-dicts detection
+        # below because columns-oriented JSON is also represented as a
+        # dictionary of dictionaries.
         if isinstance(parsed, dict) and {"columns", "data"} <= parsed.keys():
             return pd.read_json(path, orient="split")
+
+        # Detect index-oriented JSON.
+        #
+        # pandas' "columns" and "index" orientations both use a
+        # dict-of-dicts representation, so their shapes are inherently
+        # ambiguous in JSON.
+        #
+        # Heuristic: when the dict-of-dicts is scalar and rectangular, prefer
+        # the orientation implied by which dimension is larger (rows vs
+        # columns). For square cases, use value type-homogeneity to prefer
+        # columns-orient (columns are often homogeneous; rows often mix types).
+        if isinstance(parsed, dict) and parsed and all(
+            isinstance(value, dict) for value in parsed.values()
+        ):
+            inner_key_sets = [set(row.keys()) for row in parsed.values()]
+
+            same_inner_keys = len({frozenset(keys) for keys in inner_key_sets}) == 1
+
+            if same_inner_keys:
+                inner_keys = inner_key_sets[0]
+
+                cells_are_scalar = all(
+                    not isinstance(cell, (dict, list))
+                    for row in parsed.values()
+                    for cell in row.values()
+                )
+
+                if cells_are_scalar:
+                    outer_n = len(parsed)
+                    inner_n = len(inner_keys)
+
+                    if outer_n != inner_n:
+                        orient = "index" if outer_n > inner_n else "columns"
+                        return pd.read_json(path, orient=orient)
+
+                    # Square case: prefer columns-orient when values are more type-homogeneous
+                    # by column than by row.
+                    row_type_score = sum(
+                        len({type(cell) for cell in row.values()})
+                        for row in parsed.values()
+                    )
+                    col_type_score = sum(
+                        len({type(parsed[row_key][col_key]) for row_key in parsed})
+                        for col_key in inner_keys
+                    )
+
+                    orient = "index" if row_type_score > col_type_score else "columns"
+                    return pd.read_json(path, orient=orient)
         return pd.read_json(path)
 
     if suffix == ".parquet":
@@ -84,4 +133,5 @@ def get_xlsx_sheet_info(path: str) -> tuple[str, list[str]] | None:
         return None
     if not book.sheet_names:
         return None
-    return book.sheet_names[0], book.sheet_names[1:]
+
+    return book.sheet_names[0], book.sheet_names[1:]

tests/test_json_edge_cases.py

@@ -9,6 +9,7 @@
 import subprocess
 from pathlib import Path
 
+import pandas as pd
 import pytest
 
 from faircode.loaders_extra import read_table
@@ -101,3 +102,23 @@ def test_deeply_nested_json_python_current_behavior(tmp_path):
     path = _write_json(tmp_path, json.dumps({"a": {"b": {"c": 1}}}))
     df = read_table(str(path))
     assert df.to_dict(orient="records") == [{"a": {"c": 1}}]
+
+
+# ── Index-oriented JSON ──────────────────────────────────────────────────────
+def test_index_oriented_json_preserves_dataframe_shape(tmp_path):
+    original = pd.DataFrame(
+        {"sex": ["M", "F"], "age": [30, 40]},
+        index=["a", "b"],
+    )
+
+    index_path = tmp_path / "index.json"
+    original.to_json(index_path, orient="index")
+    loaded_index = read_table(str(index_path))
+    pd.testing.assert_frame_equal(loaded_index, original)
+
+    # Ensure columns-oriented dict-of-dicts with a non-default index isn't
+    # misclassified as index-oriented and transposed.
+    columns_path = tmp_path / "columns.json"
+    original.to_json(columns_path, orient="columns")
+    loaded_columns = read_table(str(columns_path))
+    pd.testing.assert_frame_equal(loaded_columns, original)