From b6e3d1a0792159142721991ccf9ffd88a6b4b6b8 Mon Sep 17 00:00:00 2001 From: Dusten Hubbard Date: Thu, 23 Jul 2026 15:24:40 -0500 Subject: [PATCH] fix(cleanup): pixel-dust threshold in pixels The data clean-up "Remove pixel-dust traces" tool now takes its size threshold in pixels (px^2) instead of physical area (um^2), matching the "pixel dust" mental model of "smaller than N pixels on its own image". findPixelDustTraces derives the physical cutoff per section from that section's magnification (threshold_px * section.mag**2), so one pixel threshold adapts to sections of different scale. Each candidate carries both its pixel area and physical area. The review dialog shows an "Area (px^2)" column (same units as the threshold) alongside "Area (um^2)" for the physical equivalent. The dialog input defaults to 10 px^2 (a stray speck of a few pixels). Tests updated to px^2 threshold semantics, including a mixed-magnification case proving the cutoff adapts per section. WHATS_NEW updated. --- PyReconstruct/modules/datatypes/series.py | 52 +++++--- .../modules/gui/dialog/malformed_contours.py | 26 ++-- PyReconstruct/modules/gui/main/main_window.py | 16 ++- WHATS_NEW.md | 2 +- tests/test_data_cleanup.py | 111 ++++++++++++++---- 5 files changed, 154 insertions(+), 53 deletions(-) diff --git a/PyReconstruct/modules/datatypes/series.py b/PyReconstruct/modules/datatypes/series.py index 1a71392a..35d5a573 100644 --- a/PyReconstruct/modules/datatypes/series.py +++ b/PyReconstruct/modules/datatypes/series.py @@ -1707,14 +1707,16 @@ def _traceMatchesSignature(trace, signature) -> bool: return True @staticmethod - def _cleanupRecord(obj_name, snum, index, trace, reason, area=None) -> dict: + def _cleanupRecord(obj_name, snum, index, trace, reason, area=None, + area_px=None) -> dict: """Build a clean-up candidate record. Uses the same schema smoothObject produces (so the records can be deleted with deleteMalformedTraces and shown in the review dialog): a "match" signature of color + 7-decimal-rounded points re-finds the exact - trace after the section is reloaded from disk. An optional physical - area (um^2) is carried for display in the pixel-dust review list. + trace after the section is reloaded from disk. An optional pixel area + (px^2) and physical area (um^2) are carried for display in the + pixel-dust review list. """ num_points = len(trace.points) record = { @@ -1736,6 +1738,8 @@ def _cleanupRecord(obj_name, snum, index, trace, reason, area=None) -> dict: } if area is not None: record["area"] = area + if area_px is not None: + record["area_px"] = area_px return record @staticmethod @@ -1753,28 +1757,39 @@ def _traceArea(trace, tform) -> float: _, area, _, _ = traceGeometry(pts, True) return abs(area) - def findPixelDustTraces(self, threshold_area : float, include_locked=False) -> list: - """Find tiny "pixel-dust" traces at or below an area threshold. + def findPixelDustTraces(self, threshold_px : float, include_locked=False) -> list: + """Find tiny "pixel-dust" traces at or below an area threshold in PIXELS. - A candidate is a CLOSED trace whose physical area (um^2, computed the - same way the object/trace tables report it) is greater than zero and at - or below ``threshold_area``. Zero-area / degenerate traces are left to - findEmptyTraces so the two operations stay disjoint. Open traces (lines) - have no enclosed area and are never pixel dust. Locked objects are - skipped unless ``include_locked`` is True. This only scans; nothing is - modified. Use deleteMalformedTraces to remove the chosen records. + The threshold is expressed in pixels (px^2) so "pixel dust" means + literally "smaller than N pixels on its own image". Because a section's + magnification (``section.mag``, um/px) sets how big a pixel is, the + physical cutoff is derived PER SECTION: a pixel-area threshold of + ``threshold_px`` becomes ``threshold_px * section.mag ** 2`` um^2 on that + section. A trace's physical area (um^2, computed the same way the + object/trace tables report it) is compared against that per-section + cutoff, so the same px threshold adapts to each section's scale. + + A candidate is a CLOSED trace whose physical area is greater than zero + and at or below the per-section cutoff. Zero-area / degenerate traces are + left to findEmptyTraces so the two operations stay disjoint. Open traces + (lines) have no enclosed area and are never pixel dust. Locked objects + are skipped unless ``include_locked`` is True. This only scans; nothing + is modified. Use deleteMalformedTraces to remove the chosen records. Params: - threshold_area (float): the maximum area (um^2), inclusive + threshold_px (float): the maximum area in pixels (px^2), inclusive include_locked (bool): True to also consider locked objects Returns: - (list): candidate records (see _cleanupRecord) + (list): candidate records (see _cleanupRecord); each carries both + the pixel area ("area_px") and its physical area ("area", um^2) """ candidates = [] for snum, section in self.enumerateSections( message="Scanning for pixel-dust traces...", ): tform = section.tform + mag2 = section.mag ** 2 # (um/px)^2, this section's pixel scale + threshold_um2 = threshold_px * mag2 # px^2 cutoff -> this section's um^2 for cname in section.contours: if not include_locked and self.getAttr(cname, "locked"): continue @@ -1782,11 +1797,16 @@ def findPixelDustTraces(self, threshold_area : float, include_locked=False) -> l if not trace.closed or len(trace.points) < 3: continue area = self._traceArea(trace, tform) - if 0 < area <= threshold_area: + if 0 < area <= threshold_um2: + area_px = area / mag2 if mag2 else 0.0 candidates.append(self._cleanupRecord( cname, snum, index, trace, - reason=f"Area {area:.6g} um^2 (<= {threshold_area:.6g})", + reason=( + f"Area {area_px:.6g} px^2 (<= {threshold_px:.6g}); " + f"{area:.6g} um^2" + ), area=area, + area_px=area_px, )) return candidates diff --git a/PyReconstruct/modules/gui/dialog/malformed_contours.py b/PyReconstruct/modules/gui/dialog/malformed_contours.py index 27881a57..537c220a 100644 --- a/PyReconstruct/modules/gui/dialog/malformed_contours.py +++ b/PyReconstruct/modules/gui/dialog/malformed_contours.py @@ -355,14 +355,21 @@ class PixelDustDialog(MalformedContoursDialog): """Review tiny "pixel-dust" traces before removing them. A data clean-up review list: every row is a small closed trace at or below - the area threshold the user chose. The user inspects the candidates (and can - "Go to trace" to confirm), then deselects any legitimate small trace before - "Delete selected" / "Delete all". Reuses all of the selection, navigation, - deletion (undoable), and export behaviour of MalformedContoursDialog; only - the columns (an Area column) and the explanatory heading differ. + the pixel-area threshold the user chose. The user inspects the candidates + (and can "Go to trace" to confirm), then deselects any legitimate small + trace before "Delete selected" / "Delete all". Reuses all of the selection, + navigation, deletion (undoable), and export behaviour of + MalformedContoursDialog; only the columns (a pixel-area column, plus its + physical-area equivalent) and the explanatory heading differ. + + The primary Area column is in pixels (px^2) so it reads in the same units as + the threshold the user set; the "Area (um^2)" column shows the physical + equivalent for that trace on its own section (each section's magnification + can differ), so both the "how many pixels" and "how big physically" views are + available at a glance. """ - COLUMNS = ["Object", "Section", "Area (um^2)", "Point count", + COLUMNS = ["Object", "Section", "Area (px^2)", "Area (um^2)", "Point count", "Location (x, y)", "Reason"] WINDOW_TITLE = "Remove pixel-dust traces" @@ -370,6 +377,7 @@ def _columnSpecs(self): return [ ("name", "str"), ("section", "int"), + ("area_px", "float"), ("area", "float"), ("points", "int"), ("location", "loc"), @@ -391,7 +399,11 @@ def _headingText(self): return ( f"{num_traces} small (pixel-dust) {trace_word} across " - f"{num_objs} {obj_word} at or below the area threshold.\n\n" + f"{num_objs} {obj_word} at or below the pixel-area threshold.\n\n" + "The Area is shown in pixels (px^2) — the same units as the " + "threshold — with the physical area (um^2) alongside; because each " + "section's magnification can differ, the same pixel size is a " + "different physical size on different sections.\n\n" "These are typically stray specks left by segmentation. Review the " "candidates below — select a row and click “Go to trace” to inspect " "one — and deselect any legitimate trace you want to keep. Then use " diff --git a/PyReconstruct/modules/gui/main/main_window.py b/PyReconstruct/modules/gui/main/main_window.py index 66546349..09d5282e 100644 --- a/PyReconstruct/modules/gui/main/main_window.py +++ b/PyReconstruct/modules/gui/main/main_window.py @@ -2621,14 +2621,20 @@ def removePixelDustTraces(self): """Find tiny "pixel-dust" traces and remove them through a review list. Scans the series for small closed traces at or below a user-chosen area - threshold (um^2), then opens a reviewable list: nothing is removed until - the user inspects the candidates, deselects any to keep, and deletes. + threshold in PIXELS (px^2), then opens a reviewable list: nothing is + removed until the user inspects the candidates, deselects any to keep, + and deletes. The pixel threshold is turned into a physical (um^2) cutoff + per section using that section's magnification, so "pixel dust" means + "smaller than N pixels on its own image" regardless of section scale. Deletion is a single undoable operation (see deleteMalformedContours). """ self.saveAllData() + # Default 10 px^2: a stray speck of a few pixels (up to roughly a 3x3 + # blob) is the target, small enough to leave genuine fine structures + # alone. Adjustable below. structure = [ - ["Maximum trace area (um^2):", ("float", 0.01)], + ["Maximum trace area (pixels, px^2):", ("float", 10.0)], ] response, confirmed = QuickDialog.get( self, structure, "Remove pixel-dust traces" @@ -2637,7 +2643,7 @@ def removePixelDustTraces(self): return threshold = response[0] if threshold is None or threshold <= 0: - notify("Please enter an area threshold greater than zero.") + notify("Please enter a pixel area threshold greater than zero.") return # locked objects are always left alone: the review-list delete path @@ -2645,7 +2651,7 @@ def removePixelDustTraces(self): # here would be a dead end. Empty-trace removal skips locked the same way. candidates = self.series.findPixelDustTraces(threshold) if not candidates: - notify("No pixel-dust traces found at or below that area.") + notify("No pixel-dust traces found at or below that pixel area.") return # reviewable list: the user confirms/deselects before anything is deleted diff --git a/WHATS_NEW.md b/WHATS_NEW.md index 026511a8..7f3bc250 100644 --- a/WHATS_NEW.md +++ b/WHATS_NEW.md @@ -6,7 +6,7 @@ full release notes on GitHub (linked from the dialog). ## [Unreleased] -- **New "Clean up" tools tidy stray traces in your series.** Under the Series menu, a new "Clean up" submenu can remove duplicate traces, find tiny "pixel-dust" specks below an area you choose (shown in a list you review and trim before anything is deleted), and remove empty traces that hold no real shape. Every clean-up is a single step you can undo (Ctrl+Z), and locked objects are left untouched. +- **New "Clean up" tools tidy stray traces in your series.** Under the Series menu, a new "Clean up" submenu can remove duplicate traces, find tiny "pixel-dust" specks below a size in pixels you choose (shown in a list you review and trim before anything is deleted, with each speck's pixel and physical area), and remove empty traces that hold no real shape. Every clean-up is a single step you can undo (Ctrl+Z), and locked objects are left untouched. - **The "Developer" update channel has been removed.** Updates now come on two channels, Stable and Beta, and Beta remains the right home for testers. If you were on Developer, PyReconstruct now keeps you on Beta automatically, so you do not need to change anything. To follow the very latest code between betas, install from source and run "git pull" (see the README). ## [1.21.0-beta-4] — 2026-07-18 diff --git a/tests/test_data_cleanup.py b/tests/test_data_cleanup.py index 6eca9ebf..1220738a 100644 --- a/tests/test_data_cleanup.py +++ b/tests/test_data_cleanup.py @@ -8,8 +8,10 @@ false positive of merging two *distinct* objects that happen to coincide. * Remove pixel-dust traces -> Series.findPixelDustTraces (scan) + a review list that deletes via Series.deleteMalformedTraces. Small closed traces at - or below an area threshold (um^2); large traces and zero-area/degenerate - traces are NOT flagged. + or below a pixel-area threshold (px^2); the physical (um^2) cutoff is + derived per section from that section's magnification, so the same px + threshold adapts to sections of different scale. Large traces and + zero-area/degenerate traces are NOT flagged. * Remove empty traces -> Series.findEmptyTraces (scan) + the same signature-based delete. Zero-area closed / zero-length open / no-point traces only; real traces (including small pixel-dust) are NOT flagged. @@ -87,6 +89,19 @@ def _count(series, snum, name): return len(series.loadSection(snum).contours.get(name, [])) +def _um2(series, snum, name): + """Physical area (um^2) of the first trace of an object on a section.""" + from PyReconstruct.modules.datatypes.series import Series + section = series.loadSection(snum) + return Series._traceArea(section.contours[name][0], section.tform) + + +def _px2(series, snum, name): + """Pixel area (px^2) = physical area / mag^2, for that section's mag.""" + section = series.loadSection(snum) + return _um2(series, snum, name) / (section.mag ** 2) + + # --------------------------------------------------------------------------- # pixel-dust # --------------------------------------------------------------------------- @@ -102,21 +117,20 @@ def test_pixel_dust_flags_only_small_closed_traces(tmp_path): _make(section, "LINE", [(0, 0), (0.1, 0), (0.2, 0)], closed=False) section.save() - from PyReconstruct.modules.datatypes.series import Series - reloaded = series.loadSection(snum) - dust_area = Series._traceArea(reloaded.contours["DUST"][0], reloaded.tform) - big_area = Series._traceArea(reloaded.contours["BIG"][0], reloaded.tform) - assert 0 < dust_area < big_area + dust_px = _px2(series, snum, "DUST") + big_px = _px2(series, snum, "BIG") + assert 0 < dust_px < big_px - threshold = (dust_area + big_area) / 2 + threshold = (dust_px + big_px) / 2 # pixel-area threshold records = series.findPixelDustTraces(threshold) names = {r["name"] for r in records} assert "DUST" in names assert "BIG" not in names # above threshold assert "LINE" not in names # open trace, no area - # every record carries the area used for display + a delete signature + # every record carries both the pixel area and physical area + a signature dust_rec = next(r for r in records if r["name"] == "DUST") - assert dust_rec["area"] == pytest.approx(dust_area) + assert dust_rec["area_px"] == pytest.approx(dust_px) + assert dust_rec["area"] == pytest.approx(_um2(series, snum, "DUST")) assert "match" in dust_rec and dust_rec["points"] == 4 @@ -128,15 +142,13 @@ def test_pixel_dust_threshold_is_inclusive_edge(tmp_path): _make(section, "DUST", [(0, 0), (0.1, 0), (0.1, 0.1), (0, 0.1)]) section.save() - from PyReconstruct.modules.datatypes.series import Series - area = Series._traceArea( - series.loadSection(snum).contours["DUST"][0], - series.loadSection(snum).tform, - ) - # exactly at area -> inclusive hit - assert {r["name"] for r in series.findPixelDustTraces(area)} >= {"DUST"} + area_px = _px2(series, snum, "DUST") + # exactly at the pixel area -> inclusive hit + assert {r["name"] for r in series.findPixelDustTraces(area_px)} >= {"DUST"} # strictly below -> miss - assert "DUST" not in {r["name"] for r in series.findPixelDustTraces(area * 0.99)} + assert "DUST" not in { + r["name"] for r in series.findPixelDustTraces(area_px * 0.99) + } def test_pixel_dust_scan_does_not_modify(tmp_path): @@ -174,12 +186,7 @@ def test_pixel_dust_delete_is_single_undo(tmp_path): _make(section, "DUST", [(0, 0), (0.1, 0), (0.1, 0.1), (0, 0.1)]) section.save() - from PyReconstruct.modules.datatypes.series import Series - area = Series._traceArea( - series.loadSection(snum).contours["DUST"][0], - series.loadSection(snum).tform, - ) - records = series.findPixelDustTraces(area * 1.5) + records = series.findPixelDustTraces(_px2(series, snum, "DUST") * 1.5) assert records states = _new_states(series) @@ -220,6 +227,62 @@ def test_pixel_dust_delete_across_sections_single_undo(tmp_path): f"one undo must restore dust on section {snum}" +def test_pixel_dust_threshold_adapts_per_section_mag(tmp_path): + """One px threshold adapts to each section's magnification. + + The SAME physical speck is placed on two sections whose magnifications + differ (coarse section has 2x the um/px of the fine one). Because pixel + area = physical area / mag^2, that identical speck is 4x more pixels on the + fine section than on the coarse one. A single pixel-area threshold chosen + between the two must therefore flag the speck on the fine section (it is + "many pixels" there) but not on the coarse section (it is "few pixels" + there) — proving the physical cutoff is derived per section, not globally. + """ + series = _load_series(tmp_path) + snums = sorted(series.sections) + fine_snum, coarse_snum = snums[0], snums[1] + + dust_pts = [(0, 0), (0.1, 0), (0.1, 0.1), (0, 0.1)] + fine = series.loadSection(fine_snum) + base_mag = fine.mag + _make(fine, "DUST", dust_pts) + fine.mag = base_mag # fine scale: more pixels per micron + fine.save() + + coarse = series.loadSection(coarse_snum) + _make(coarse, "DUST", list(dust_pts)) + coarse.mag = base_mag * 2 # coarse scale: fewer pixels per micron + coarse.save() + + # essentially the same physical speck on both sections (the fixture's two + # sections carry slightly different transforms, hence the loose tolerance) + assert _um2(series, fine_snum, "DUST") == pytest.approx( + _um2(series, coarse_snum, "DUST"), rel=1e-2 + ) + fine_px = _px2(series, fine_snum, "DUST") + coarse_px = _px2(series, coarse_snum, "DUST") + # the fine section renders that speck as ~4x more pixels (mag differs 2x) + assert fine_px == pytest.approx(coarse_px * 4, rel=1e-2) + + # threshold between the two pixel areas: flags the fine section only + threshold = (fine_px + coarse_px) / 2 + hits = { + (r["section"], r["name"]) + for r in series.findPixelDustTraces(threshold) + if r["name"] == "DUST" + } + assert (coarse_snum, "DUST") in hits # few pixels on the coarse section + assert (fine_snum, "DUST") not in hits # many pixels on the fine section + + # each record reports the pixel area on its OWN section's scale + rec = next( + r for r in series.findPixelDustTraces(threshold) + if r["section"] == coarse_snum and r["name"] == "DUST" + ) + assert rec["area_px"] == pytest.approx(coarse_px) + assert rec["area"] == pytest.approx(_um2(series, coarse_snum, "DUST")) + + # --------------------------------------------------------------------------- # empty / degenerate # ---------------------------------------------------------------------------