From 2b2366d63acf7f9f3f0074a73493b3f2bcaaa4e4 Mon Sep 17 00:00:00 2001 From: nitishchauhan002 Date: Wed, 2 Sep 2026 21:57:05 +0530 Subject: [PATCH 1/4] Fix index-oriented JSON parsing --- faircode/loaders_extra.py | 52 +++++++++++++++++++++++++++++------ tests/test_json_edge_cases.py | 15 ++++++++++ 2 files changed, 59 insertions(+), 8 deletions(-) diff --git a/faircode/loaders_extra.py b/faircode/loaders_extra.py index 2ff8d20..eb0de1b 100644 --- a/faircode/loaders_extra.py +++ b/faircode/loaders_extra.py @@ -36,6 +36,7 @@ def read_table(path: str) -> pd.DataFrame: if suffix == ".json": with open(path, "r", encoding="utf-8") as f: raw = f.read() + try: parsed = json.loads(raw) except json.JSONDecodeError as exc: @@ -43,15 +44,49 @@ def read_table(path: str) -> pd.DataFrame: # file) is an internal/version-specific message. Fail fast with # a clear message instead, mirroring the JS engine's parseJSON(). raise ValueError(f"Unsupported JSON format (not valid JSON: {exc}).") from exc - # Detect split-orient ({"columns": [...], "data": [...]}, optionally - # "index") from the parsed shape rather than trying the default - # (records) orientation first and catching its ValueError: a split - # file that omits the optional "index" key parses without error - # under the default orientation too - as two columns literally named - # "columns" and "data" - so the ValueError this used to rely on - # never fires, and the wrong shape comes back silently. + + # Detect split-orient JSON: + # {"columns": [...], "data": [...]}, optionally with "index". + # + # This must be checked before the generic dict-of-dicts detection + # below because columns-oriented JSON is also represented as a + # dictionary of dictionaries. if isinstance(parsed, dict) and {"columns", "data"} <= parsed.keys(): return pd.read_json(path, orient="split") + + # Detect index-oriented JSON. + # + # pandas' "columns" and "index" orientations both use a + # dict-of-dicts representation, so their shapes are inherently + # ambiguous in JSON. A columns-oriented export using the default + # DataFrame index has numeric inner keys ("0", "1", ...). Preserve + # that existing/common case and treat other scalar dict-of-dicts as + # index-oriented. + if isinstance(parsed, dict) and parsed and all( + isinstance(value, dict) for value in parsed.values() + ): + inner_key_sets = [set(row.keys()) for row in parsed.values()] + + same_inner_keys = len( + {frozenset(keys) for keys in inner_key_sets} + ) == 1 + + if same_inner_keys: + inner_keys = inner_key_sets[0] + + inner_keys_are_default_index = inner_keys == { + str(i) for i in range(len(inner_keys)) + } + + cells_are_scalar = all( + not isinstance(cell, (dict, list)) + for row in parsed.values() + for cell in row.values() + ) + + if cells_are_scalar and not inner_keys_are_default_index: + return pd.read_json(path, orient="index") + return pd.read_json(path) if suffix == ".parquet": @@ -84,4 +119,5 @@ def get_xlsx_sheet_info(path: str) -> tuple[str, list[str]] | None: return None if not book.sheet_names: return None - return book.sheet_names[0], book.sheet_names[1:] + + return book.sheet_names[0], book.sheet_names[1:] \ No newline at end of file diff --git a/tests/test_json_edge_cases.py b/tests/test_json_edge_cases.py index de63658..1e04e48 100644 --- a/tests/test_json_edge_cases.py +++ b/tests/test_json_edge_cases.py @@ -9,6 +9,7 @@ import subprocess from pathlib import Path +import pandas as pd import pytest from faircode.loaders_extra import read_table @@ -101,3 +102,17 @@ def test_deeply_nested_json_python_current_behavior(tmp_path): path = _write_json(tmp_path, json.dumps({"a": {"b": {"c": 1}}})) df = read_table(str(path)) assert df.to_dict(orient="records") == [{"a": {"c": 1}}] + + +# ── Index-oriented JSON ────────────────────────────────────────────────────── +def test_index_oriented_json_preserves_dataframe_shape(tmp_path): + original = pd.DataFrame( + {"sex": ["M", "F"], "age": [30, 40]}, + index=["a", "b"], + ) + path = tmp_path / "index.json" + original.to_json(path, orient="index") + + loaded = read_table(str(path)) + + assert loaded.equals(original) \ No newline at end of file From 322b5a734f89e9624f3e3750ea18d266f72a4ef5 Mon Sep 17 00:00:00 2001 From: Yash Kewlani <86704881+yakew7@users.noreply.github.com> Date: Wed, 2 Sep 2026 23:14:21 +0530 Subject: [PATCH 2/4] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- tests/test_json_edge_cases.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/tests/test_json_edge_cases.py b/tests/test_json_edge_cases.py index 1e04e48..a6dee72 100644 --- a/tests/test_json_edge_cases.py +++ b/tests/test_json_edge_cases.py @@ -110,9 +110,15 @@ def test_index_oriented_json_preserves_dataframe_shape(tmp_path): {"sex": ["M", "F"], "age": [30, 40]}, index=["a", "b"], ) - path = tmp_path / "index.json" - original.to_json(path, orient="index") - loaded = read_table(str(path)) - - assert loaded.equals(original) \ No newline at end of file + index_path = tmp_path / "index.json" + original.to_json(index_path, orient="index") + loaded_index = read_table(str(index_path)) + pd.testing.assert_frame_equal(loaded_index, original) + + # Ensure columns-oriented dict-of-dicts with a non-default index isn't + # misclassified as index-oriented and transposed. + columns_path = tmp_path / "columns.json" + original.to_json(columns_path, orient="columns") + loaded_columns = read_table(str(columns_path)) + pd.testing.assert_frame_equal(loaded_columns, original) \ No newline at end of file From 6c8f8f89b5601c3e10ef1bf757cf993633bb59f5 Mon Sep 17 00:00:00 2001 From: Yash Kewlani <86704881+yakew7@users.noreply.github.com> Date: Wed, 2 Sep 2026 23:20:18 +0530 Subject: [PATCH 3/4] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- faircode/loaders_extra.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/faircode/loaders_extra.py b/faircode/loaders_extra.py index eb0de1b..e1e8275 100644 --- a/faircode/loaders_extra.py +++ b/faircode/loaders_extra.py @@ -58,10 +58,12 @@ def read_table(path: str) -> pd.DataFrame: # # pandas' "columns" and "index" orientations both use a # dict-of-dicts representation, so their shapes are inherently - # ambiguous in JSON. A columns-oriented export using the default - # DataFrame index has numeric inner keys ("0", "1", ...). Preserve - # that existing/common case and treat other scalar dict-of-dicts as - # index-oriented. + # ambiguous in JSON. + # + # Heuristic: when the dict-of-dicts is scalar and rectangular, prefer + # the orientation implied by which dimension is larger (rows vs + # columns). For square cases, use value type-homogeneity to prefer + # columns-orient (columns are often homogeneous; rows often mix types). if isinstance(parsed, dict) and parsed and all( isinstance(value, dict) for value in parsed.values() ): From e8a73380308e63cbdb18f8a8a412e4a524c481a1 Mon Sep 17 00:00:00 2001 From: Yash Kewlani <86704881+yakew7@users.noreply.github.com> Date: Wed, 2 Sep 2026 23:25:05 +0530 Subject: [PATCH 4/4] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- faircode/loaders_extra.py | 32 ++++++++++++++++++++++---------- 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/faircode/loaders_extra.py b/faircode/loaders_extra.py index e1e8275..d1ec1de 100644 --- a/faircode/loaders_extra.py +++ b/faircode/loaders_extra.py @@ -69,26 +69,38 @@ def read_table(path: str) -> pd.DataFrame: ): inner_key_sets = [set(row.keys()) for row in parsed.values()] - same_inner_keys = len( - {frozenset(keys) for keys in inner_key_sets} - ) == 1 + same_inner_keys = len({frozenset(keys) for keys in inner_key_sets}) == 1 if same_inner_keys: inner_keys = inner_key_sets[0] - inner_keys_are_default_index = inner_keys == { - str(i) for i in range(len(inner_keys)) - } - cells_are_scalar = all( not isinstance(cell, (dict, list)) for row in parsed.values() for cell in row.values() ) - if cells_are_scalar and not inner_keys_are_default_index: - return pd.read_json(path, orient="index") - + if cells_are_scalar: + outer_n = len(parsed) + inner_n = len(inner_keys) + + if outer_n != inner_n: + orient = "index" if outer_n > inner_n else "columns" + return pd.read_json(path, orient=orient) + + # Square case: prefer columns-orient when values are more type-homogeneous + # by column than by row. + row_type_score = sum( + len({type(cell) for cell in row.values()}) + for row in parsed.values() + ) + col_type_score = sum( + len({type(parsed[row_key][col_key]) for row_key in parsed}) + for col_key in inner_keys + ) + + orient = "index" if row_type_score > col_type_score else "columns" + return pd.read_json(path, orient=orient) return pd.read_json(path) if suffix == ".parquet":