diff --git a/ui/src/Components/InteractiveLabeler/InteractiveLabeler.jsx b/ui/src/Components/InteractiveLabeler/InteractiveLabeler.jsx index 329bb33..93a271e 100644 --- a/ui/src/Components/InteractiveLabeler/InteractiveLabeler.jsx +++ b/ui/src/Components/InteractiveLabeler/InteractiveLabeler.jsx @@ -26,6 +26,9 @@ import { DefaultButton, PrimaryButton, ProgressIndicator, + Separator, + Spinner, + SpinnerSize, Text, Toggle, } from "@fluentui/react"; @@ -43,6 +46,7 @@ import { CLASS_DAMAGED, CLASS_INTACT, OvRLogisticRegression, + crossValidateMetrics, holdoutMetricsDamaged, isValidVector, } from "./interactiveModel.js"; @@ -103,6 +107,8 @@ async function fetchArtifactBuffer(url) { // Class colors (match index.html). Index = class number. const CLASS_COLORS = ["#107C10", "#C50F1F", "#5B5FC7"]; // intact, damaged, cloudy +// Human-readable class names, indexed by class number (parallel to CLASS_COLORS). +const CLASS_LABELS = ["Intact", "Damaged", "Cloudy"]; const UNLABELED_COLOR = "#BDBDBD"; // In-browser class -> validation-report vocabulary (Damaged/NotDamaged/Unknown). @@ -177,6 +183,53 @@ function fillOpacityExpr(stateKey) { ]; } +// ── Uncertainty view ──────────────────────────────────────────────────────── +// Colors every scored building by the model's predictive uncertainty (0 = +// confident, 1 = maximally uncertain). Stops match UNCERTAINTY_LEGEND_GRADIENT +// used by the panel legend. The driver reads the "unc" feature-state; buildings +// without a computed value coalesce to 0 (and are hidden by the opacity expr). +const UNCERTAINTY_STOPS = [ + [0, "#2c7bb6"], + [0.25, "#abd9e9"], + [0.5, "#ffffbf"], + [0.75, "#fdae61"], + [1, "#d7191c"], +]; +const UNCERTAINTY_LEGEND_GRADIENT = + "linear-gradient(to right, #2c7bb6, #abd9e9, #ffffbf, #fdae61, #d7191c)"; +const UNCERTAINTY_FILL_OPACITY = 0.6; + +function fillColorExprUncertainty() { + return [ + "interpolate", + ["linear"], + ["coalesce", ["feature-state", "unc"], 0], + ...UNCERTAINTY_STOPS.flat(), + ]; +} + +// Only buildings with a computed "unc" value are painted; the rest stay +// transparent (coalesce sentinel -1 marks "no value"). +function fillOpacityExprUncertainty() { + return [ + "case", + ["==", ["coalesce", ["feature-state", "unc"], -1], -1], 0, + UNCERTAINTY_FILL_OPACITY, + ]; +} + +// Normalized Shannon entropy of a probability vector, in [0, 1] (0 = a single +// class has all the mass, 1 = uniform across k classes). Used as the per- +// building uncertainty score for the uncertainty view. +function normalizedEntropy(probs) { + const k = probs.length; + if (k <= 1) return 0; + let h = 0; + for (const p of probs) if (p > 0) h -= p * Math.log(p); + const norm = h / Math.log(k); + return Math.max(0, Math.min(1, norm)); +} + // Binary HFTR sidecar: // bytes 0-3 : magic "HFTR" // bytes 4-7 : u32 LE version (currently 1) @@ -231,6 +284,18 @@ async function fetchFeaturesSidecar(url) { return { matrix, n, d }; } +// Format a { mean, std } CV-metric block for the Advanced results table. +// asPercent=true renders precision/recall as "92 ± 3%"; otherwise (AUC) as +// two-decimal "0.92 ± 0.03". Returns an em-dash when the metric is undefined +// (mean == null, e.g. no fold produced a defined value). +function fmtMetric(m, asPercent) { + if (!m || m.mean == null) return "—"; + if (asPercent) { + return `${(m.mean * 100).toFixed(0)} ± ${(m.std * 100).toFixed(0)}%`; + } + return `${m.mean.toFixed(2)} ± ${m.std.toFixed(2)}`; +} + const InteractiveLabeler = () => { const { projectId, imageLayerId, modelId } = useParams(); const navigate = useNavigate(); @@ -279,6 +344,22 @@ const InteractiveLabeler = () => { const labelsDirtyRef = useRef(true); const fullPredictAbortRef = useRef({ cancelled: false }); + // Advanced → Swipe view. atlas.SwipeMap always reveals its SECONDARY map on + // the RIGHT of the divider and shows its PRIMARY on the LEFT, so to land PRE + // imagery on the left / POST imagery on the right we make the labeler map + // (mapRef.current: post-event imagery + footprints + interaction) the + // SECONDARY and a freshly-built pre-event map the PRIMARY. swipePreMapRef + // holds that new pre-event map (created in swipeMapContainerRef), swipeRef + // holds the atlas.SwipeMap that draws the divider, layerImageryRef caches the + // layer's imagery URLs (resolved in createMap), and swipePmtilesUrlRef caches + // the PMTiles archive URL so the pre map can draw the same building + // footprints from the same source. + const swipeMapContainerRef = useRef(null); + const swipePreMapRef = useRef(null); + const swipeRef = useRef(null); + const layerImageryRef = useRef(null); + const swipePmtilesUrlRef = useRef(null); + const [isMapReady, setIsMapReady] = useState(false); const [selectedClass, setSelectedClass] = useState(CLASS_DAMAGED); const [viewMode, setViewMode] = useState("label"); // "label" | "predict" @@ -292,6 +373,14 @@ const InteractiveLabeler = () => { // Progress state for the "Predict all buildings" full-coverage pass. const [fullPredict, setFullPredict] = useState(null); const [backend, setBackend] = useState(null); + // Advanced section: expand/collapse, 5-fold CV result + busy flag, swipe view. + const [advancedOpen, setAdvancedOpen] = useState(false); + const [cvResult, setCvResult] = useState(null); + const [cvRunning, setCvRunning] = useState(false); + const [swipeOn, setSwipeOn] = useState(false); + // Advanced → Uncertainty view: recolor every scored footprint by the model's + // predictive uncertainty (with a legend). + const [uncertaintyOn, setUncertaintyOn] = useState(false); // selectedClass / viewMode are read by long-lived map event handlers. const selectedClassRef = useRef(selectedClass); @@ -302,6 +391,15 @@ const InteractiveLabeler = () => { useEffect(() => { viewModeRef.current = viewMode; }, [viewMode]); + // Read by long-lived map handlers (hydrateViewport) to decide whether to + // compute per-building uncertainty on each viewport settle. + const uncertaintyOnRef = useRef(uncertaintyOn); + useEffect(() => { + uncertaintyOnRef.current = uncertaintyOn; + }, [uncertaintyOn]); + // Read by the P hotkey (long-lived listener) so it can't switch to the + // Predicted view before there are enough labels to train. + const canTrainRef = useRef(false); // Detect the compute backend up-front so the panel shows WebGPU vs CPU. useEffect(() => { @@ -352,6 +450,10 @@ const InteractiveLabeler = () => { } catch { // Imagery is optional — labeling works without it. } + // Cache the imagery URLs for the Advanced → Swipe view, which loads the + // pre-event tiles onto its secondary map (falls back to satellite when + // the layer has no pre-event imagery). + layerImageryRef.current = layerData?.imagery || null; // Resolve the model's PMTiles URL. Models are returned by // GetLayerModelsDetails; pick ours by modelId. The pmtilesUrl is @@ -512,6 +614,11 @@ const InteractiveLabeler = () => { // then overzooms tiles at z>maxZoom automatically. Setting // maxSourceZoom explicitly capped at 14 makes Atlas treat z>14 as // "source has no data" and stop rendering past that zoom. + // Cache the PMTiles archive URL so the Advanced → Swipe pre map can draw + // the same building footprints from the same source (see the swipe + // effect below). Must match this source's `pmtiles://` exactly so + // both maps route through the same in-memory pmtiles handle. + swipePmtilesUrlRef.current = browserPmtilesUrl; const source = new window.atlas.source.VectorTileSource("buildings", { type: "vector", url: `pmtiles://${browserPmtilesUrl}`, @@ -733,6 +840,65 @@ const InteractiveLabeler = () => { if (viewModeRef.current === "predict") { maybeTrainAndPredict(features); } + + // Uncertainty view — recolor the on-screen buildings by model uncertainty. + if (uncertaintyOnRef.current) { + computeUncertaintyForViewport(features); + } + } + + // Train (or reuse the cached model) and paint per-building uncertainty as the + // "unc" feature-state for every building currently in the viewport. Shares + // trainedModelRef with the predict path; returns silently (with a status + // hint) when there aren't enough labels to train. + function computeUncertaintyForViewport(features) { + const entries = Object.values(labeledMapRef.current).filter((e) => + isValidVector(e.features) + ); + const perClass = {}; + entries.forEach((e) => (perClass[e.label] = (perClass[e.label] || 0) + 1)); + const classesReady = Object.values(perClass).filter( + (n) => n >= MIN_PER_CLASS + ).length; + if (classesReady < 2) { + setStatus( + `Need ${MIN_PER_CLASS}+ labels in at least 2 classes for uncertainty.` + ); + return; + } + // Reuse the cached model unless the label set changed since it was trained. + if (labelsDirtyRef.current || !trainedModelRef.current) { + const ovr = new OvRLogisticRegression({ + learningRate: 0.1, + numSteps: 500, + lambda: 0.01, + }); + ovr.train( + entries.map((e) => e.features), + entries.map((e) => e.label) + ); + trainedModelRef.current = ovr; + labelsDirtyRef.current = false; + } + const model = trainedModelRef.current; + + const ids = []; + const matrix = []; + const sources = []; + for (const f of features) { + if (f.id == null) continue; + const vec = lookupFeatureVector(f.id); + if (!isValidVector(vec)) continue; + ids.push(f.id); + matrix.push(vec); + sources.push(f.source); + } + if (matrix.length === 0) return; + const probs = model.predictProba(matrix); + for (let i = 0; i < ids.length; i++) { + const unc = normalizedEntropy(Object.values(probs[i])); + setFeatureStateUnc(sources[i], ids[i], unc); + } } // ── feature-state helpers (drive renderer paint) ────────────────────────── @@ -762,6 +928,19 @@ const InteractiveLabeler = () => { console.warn("setFeatureState (pred) failed:", err); } } + function setFeatureStateUnc(sourceId, id, unc) { + const gl = glMapRef.current; + if (!gl) return; + try { + gl.setFeatureState( + { source: sourceId, sourceLayer: PMTILES_SOURCE_LAYER, id }, + { unc } + ); + mapRef.current?.triggerRepaint && mapRef.current.triggerRepaint(); + } catch (err) { + console.warn("setFeatureState (unc) failed:", err); + } + } function clearFeatureStateLabel(sourceId, id) { const gl = glMapRef.current; if (!gl) return; @@ -791,12 +970,21 @@ const InteractiveLabeler = () => { if (!map) return; const fill = map.layers.getLayerById?.("embeddingFill"); if (!fill) return; - fill.setOptions({ - fillColor: fillColorExpr(viewMode === "predict" ? "pred" : "label"), - fillOpacity: fillOpacityExpr(viewMode === "predict" ? "pred" : "label"), - }); - if (viewMode === "predict") hydrateViewport(map); - }, [viewMode, isMapReady]); + if (uncertaintyOn) { + // Uncertainty view overrides the label/predict coloring entirely. + fill.setOptions({ + fillColor: fillColorExprUncertainty(), + fillOpacity: fillOpacityExprUncertainty(), + }); + hydrateViewport(map); + } else { + fill.setOptions({ + fillColor: fillColorExpr(viewMode === "predict" ? "pred" : "label"), + fillOpacity: fillOpacityExpr(viewMode === "predict" ? "pred" : "label"), + }); + if (viewMode === "predict") hydrateViewport(map); + } + }, [viewMode, uncertaintyOn, isMapReady]); // Show / hide the buildings layers without unmounting them. Driven by // the panel toggle + spacebar hotkey; the feature-state and the cached @@ -1056,19 +1244,241 @@ const InteractiveLabeler = () => { else if (e.key === "3") setSelectedClass(CLASS_CLOUDY); else if (e.key === "t" || e.key === "T") setSelectedClass((c) => (c + 1) % 3); - else if (e.key === "p" || e.key === "P") - setViewMode((v) => (v === "label" ? "predict" : "label")); - else if (e.key === " " || e.code === "Space") { + else if (e.key === "p" || e.key === "P") { + // Predicted view needs a trained model; ignore until we have enough + // labels (matches the disabled View toggle). + if (!canTrainRef.current) return; + const next = viewModeRef.current === "label" ? "predict" : "label"; + setViewMode(next); + // Predicted view and Uncertainty view are mutually exclusive. + if (next === "predict") setUncertaintyOn(false); + } else if (e.key === " " || e.code === "Space") { // preventDefault to stop the browser from scrolling the page // when the map container doesn't have focus. e.preventDefault(); setShowFootprints((v) => !v); + } else if ( + swipeRef.current && + (e.key === "a" || e.key === "s" || e.key === "d") + ) { + // Snap the swipe divider: 'a' = full left, 's' = middle, 'd' = full + // right. sliderPosition is in pixels from the left of the map; the + // SwipeMap module clamps out-of-range values to [0, width]. + const el = swipeMapContainerRef.current || mapContainerRef.current; + const w = el ? el.getBoundingClientRect().width : 0; + const pos = e.key === "a" ? 0 : e.key === "s" ? w / 2 : w; + try { + swipeRef.current.setOptions({ sliderPosition: pos }); + } catch (err) { + console.warn("swipe setOptions (sliderPosition) failed:", err); + } } } window.addEventListener("keydown", onKeyDown); return () => window.removeEventListener("keydown", onKeyDown); }, []); + // ── Advanced → 5-fold cross-validation ──────────────────────────────────── + // Runs stratified k-fold CV over the current labels via crossValidateMetrics. + // It's CPU-bound, so crossValidateMetrics is async and yields between folds — + // this keeps the main thread free enough that the map keeps rendering instead + // of blanking out while the CV runs. + async function handleRunCV() { + const entries = Object.values(labeledMapRef.current).filter((e) => + isValidVector(e.features) + ); + if (entries.length === 0) { + setCvResult({ ok: false, reason: "Label some buildings first." }); + return; + } + setCvRunning(true); + setCvResult(null); + await new Promise((resolve) => setTimeout(resolve, 0)); + try { + setCvResult(await crossValidateMetrics(entries, { k: 5 })); + } catch (e) { + console.error("Cross-validation failed:", e); + setCvResult({ + ok: false, + reason: `Cross-validation failed: ${e?.message || e}`, + }); + } finally { + setCvRunning(false); + } + } + + // ── Advanced → Swipe view (pre-event imagery reveal) ────────────────────── + // Mirrors the Visualizer's swipe pattern (Visualizer.jsx:140-155, 466-467) + // using the global atlas.SwipeMap loaded from + // /assets/js/azure-maps-swipe-map.min.js in index.html — NOT an npm import. + // + // atlas.SwipeMap always clips its SECONDARY map to reveal it on the RIGHT of + // the divider and shows its PRIMARY on the LEFT, and syncs both cameras on + // every 'move'. To land PRE imagery on the left / POST imagery on the right + // we therefore wire: + // • PRIMARY = a freshly-built PRE-event map (created in + // swipeMapContainerRef, the FIRST/behind map div), and + // • SECONDARY = the existing labeler map (mapRef.current: post-event + // imagery + building footprints + click/label interaction), + // which sits in the SECOND/on-top map div so its clipped + // right half reveals the pre map underneath on the left. + // The labeler map is ADOPTED as the secondary — SwipeMap only adds 'move' + + // 'resize' handlers and clips its container, so an already-"ready" map + // adopts cleanly and its own click/label/hydrate handlers are untouched. On + // teardown we dispose ONLY the swipe control + the new pre map and clear the + // clip SwipeMap left on the labeler's container; the labeler map itself is + // never disposed here, so toggling swipe off/on repeatedly leaves the + // labeler and its footprint interactions fully intact. + useEffect(() => { + if (!isMapReady || !swipeOn) return undefined; + const labelerMap = mapRef.current; + const container = swipeMapContainerRef.current; + // Capture the labeler map's container node up front so the cleanup below + // does not read a ref (mapContainerRef.current) that may have changed by + // teardown — the node is stable for this effect's lifetime. + const labelerContainer = mapContainerRef.current; + if (!labelerMap || !container || !window.atlas || !window.atlas.SwipeMap) { + return undefined; + } + + // Seed the new pre map with the labeler's current camera so the two start + // aligned before SwipeMap takes over camera synchronization. + const cam = labelerMap.getCamera(); + const preMap = new window.atlas.Map(container, { + center: cam.center, + zoom: cam.zoom, + bearing: cam.bearing || 0, + pitch: cam.pitch || 0, + maxPitch: 0, + // Match the labeler map's style handling: "satellite" shows the Azure + // aerial basemap in real deployments, while local docker dev (no Azure + // Maps subscription) uses "blank" so the map control still fires "ready" + // without a valid token. The pre-event TileLayer is added on top in + // either case; with no pre-event imagery, loadImagery("") falls back to + // the Azure satellite tileset (which renders once a real token exists). + style: isAzureMapsPlaceholder ? "blank" : "satellite", + language: "en-US", + authOptions: getAzureMapsAuthOptions(), + }); + swipePreMapRef.current = preMap; + + preMap.events.add("ready", () => { + // Match the labeler map's interaction constraints (no rotate / pitch). + preMap.setUserInteraction({ + dragRotateInteraction: false, + scrollZoomInteraction: true, + pinchZoomInteraction: true, + pinchRotateInteraction: false, + }); + // Pre-event imagery on the LEFT (primary) pane. loadImagery falls back to + // the Azure satellite tileset when the tile URL is "" (no pre-event + // imagery). + const preUrl = layerImageryRef.current?.preEventTileUrl || ""; + loadImagery( + toBrowserTitilerUrl(preUrl), + preMap, + { current: null }, + "swipePreEventLayer", + true + ); + + // Draw the building footprints on the pre map too, from the SAME PMTiles + // archive + styling helpers the labeler uses, so outlines (and the + // unlabeled fill) span BOTH panes continuously — SwipeMap clips an entire + // map, so a single footprint layer can only appear on one side. + // + // NOTE: per-building label colors are driven by feature-state on the + // labeler's internal GL map (glMapRef), which this pre map does not have, + // so footprints render here in the UNLABELED color while still showing + // outlines. Per-building label colors currently render on the + // post/interactive (right) pane only; mirroring feature-state across the + // two GL maps would be a follow-up. + if (swipePmtilesUrlRef.current) { + try { + preMap.sources.add( + new window.atlas.source.VectorTileSource("buildings", { + type: "vector", + url: `pmtiles://${swipePmtilesUrlRef.current}`, + }) + ); + preMap.layers.add( + new window.atlas.layer.PolygonLayer("buildings", "swipeFill", { + sourceLayer: PMTILES_SOURCE_LAYER, + fillColor: UNLABELED_COLOR, + fillOpacity: 0.15, + }) + ); + preMap.layers.add( + new window.atlas.layer.LineLayer("buildings", "swipeOutline", { + sourceLayer: PMTILES_SOURCE_LAYER, + strokeColor: "#1a5276", + minZoom: 15, + strokeWidth: ["step", ["zoom"], 1, 16, 2], + }) + ); + } catch (e) { + console.warn("Swipe pre-map footprints failed:", e); + } + } + + // Wire the native swipe control: PRIMARY = pre map (revealed on the + // LEFT), SECONDARY = labeler map (clipped, revealed on the RIGHT). + // atlas.SwipeMap keeps BOTH cameras in sync on every 'move' internally, + // so we must NOT add our own camera-sync handler (doing so + // double-updates the cameras and makes panning jump/stutter). + try { + swipeRef.current = new window.atlas.SwipeMap(preMap, labelerMap); + } catch (e) { + console.warn("atlas.SwipeMap init failed:", e); + } + }); + + return () => { + // Tear down the swipe control first — its dispose() removes the divider + // handle it appended to the PRIMARY (pre map) container and detaches the + // 'move'/'resize' sync handlers from BOTH maps — then dispose the new pre + // map. The labeler map (mapRef.current / the SECONDARY) is deliberately + // NOT disposed so its footprint interactions survive toggling swipe off. + if (swipeRef.current) { + try { + if (typeof swipeRef.current.dispose === "function") { + swipeRef.current.dispose(); + } + } catch (e) { + console.warn("atlas.SwipeMap dispose failed:", e); + } + swipeRef.current = null; + } + if (swipePreMapRef.current) { + try { + swipePreMapRef.current.dispose(); + } catch (e) { + console.warn("swipe pre map dispose failed:", e); + } + swipePreMapRef.current = null; + } + // SwipeMap.dispose() does NOT clear the inline `clip` it set on the + // SECONDARY (labeler) map's container, so clear it here or the labeler + // stays clipped to its right half after swipe is turned off. Clear it on + // both the element getMapContainer() reports and the div we passed to the + // Map constructor, to be safe across Atlas builds. + try { + if (labelerMap && typeof labelerMap.getMapContainer === "function") { + labelerMap.getMapContainer().style.clip = ""; + } + } catch (e) { + console.warn("clearing labeler map clip failed:", e); + } + if (labelerContainer) { + labelerContainer.style.clip = ""; + } + // Reset the pre map's container so the next toggle starts from a clean + // slate (any leftover atlas DOM / clip is removed). + container.style.clip = ""; + container.innerHTML = ""; + }; + }, [isMapReady, swipeOn]); + // ── Save labels ─────────────────────────────────────────────────────────── // Persists the manual labels to the model-scoped interactive-labeler store. // Predictions are persisted by the separate "Predict all buildings" flow @@ -1328,12 +1738,37 @@ const InteractiveLabeler = () => { } const totalLabeled = counts[0] + counts[1] + counts[2]; + // The Predicted and Uncertainty views both need a trained model, which needs + // at least MIN_PER_CLASS labels in 2+ classes. Gate both toggles on this. + const canTrain = + [ + counts[CLASS_INTACT], + counts[CLASS_DAMAGED], + counts[CLASS_CLOUDY], + ].filter((n) => n >= MIN_PER_CLASS).length >= 2; + + // If labels drop back below the trainable threshold (e.g. after clearing), + // fall back to the Labeled view so we don't sit in a now-disabled Predicted + // or Uncertainty view with no model behind it. + useEffect(() => { + canTrainRef.current = canTrain; + if (!canTrain) { + setUncertaintyOn(false); + setViewMode("label"); + } + }, [canTrain]); return (
{
{
-
+ {/* Map area: the labeler map plus the Advanced → Swipe view. Both map + divs are absolutely positioned and fill this relative wrapper exactly + (the same overlapping layout the Visualizer's swipe uses — see + Visualizer.jsx + visualizer.css `.map`). atlas.SwipeMap reveals its + SECONDARY on the RIGHT and shows its PRIMARY on the LEFT, so to land + PRE imagery on the left / POST imagery on the right the PRE map + (primary) must sit in the FIRST/behind div and the labeler map (post + + footprints, secondary) in the SECOND/on-top div — its clipped right + half then reveals the pre map underneath on the left. The divider + handle (z-index:1, appended into the primary/pre container) still + paints above both. The pre-map overlay stays display:none until the + swipe toggle is on. */} +
+ {/* FIRST/behind: swipe PRIMARY = the new pre-event map (built on + demand by the swipe effect while the toggle is on). */} +
+ {/* SECOND/on-top: the labeler map. When swipe is on it is adopted as + the SwipeMap SECONDARY and clipped to its right half; when swipe is + off it is the only visible map. */} +
+ + {/* Pane labels, shown only while swipe is on. Rendered inside this + relative wrapper so left/right map to the map area (not the window). + "Pre imagery" sits at the very top-left (above the Back button, which + drops to top:46 while swiping); "Post imagery" hugs the top-right + corner. pointerEvents: none so they never intercept a divider + drag. */} + {swipeOn && ( + <> +
+ Pre imagery +
+
+ Post imagery +
+ + )} + + {/* Legend, bottom-right of the map. Shows the class colors normally + (Intact / Damaged / Cloudy), or the uncertainty ramp when the + uncertainty view is on. Hidden when footprints are hidden. */} + {isMapReady && showFootprints && ( +
+ {uncertaintyOn ? ( + <> +
+ Model uncertainty +
+
+
+ Low (confident) + High +
+ + ) : ( + <> +
+ {viewMode === "predict" ? "Predicted" : "Labels"} +
+ {CLASS_LABELS.map((name, i) => ( +
+ + {name} +
+ ))} + + )} +
+ )} +
{isMapReady && (
{ padding: 16, background: "#fff", borderLeft: "1px solid #e1e1e1", - overflowY: "auto", + display: "flex", + flexDirection: "column", }} > +
Interactive Labeler @@ -1413,11 +2002,20 @@ const InteractiveLabeler = () => { onText="Predicted" offText="Labeled" checked={viewMode === "predict"} - onChange={(e, checked) => - setViewMode(checked ? "predict" : "label") - } + disabled={!canTrain} + onChange={(e, checked) => { + setViewMode(checked ? "predict" : "label"); + // Predicted view and Uncertainty view are mutually exclusive. + if (checked) setUncertaintyOn(false); + }} style={{ marginTop: 12 }} /> + {!canTrain && ( +
+ Predicted / Uncertainty views need {MIN_PER_CLASS}+ labels in at + least 2 classes. +
+ )} { title="Remove every label for this model — both in-session and in the saved store." /> -
+ {/* Advanced: expandable container for the 5-fold CV report and the + swipe (pre-event) comparison view. */} + setAdvancedOpen((v) => !v)} + styles={{ root: { marginTop: 12, paddingLeft: 0, height: 28 } }} + > + Advanced + + + {advancedOpen && ( +
+ + + {cvRunning && ( +
+ +
+ )} + + {cvResult && !cvResult.ok && ( +
+ {cvResult.reason} +
+ )} + + {cvResult && cvResult.ok && ( +
+ + + + + + + + + + + {cvResult.classes.map((c) => { + const pc = cvResult.perClass[c]; + return ( + + + + + + + ); + })} + +
+ Class + + P + + R + + AUC +
+ {CLASS_LABELS[c] ?? `Class ${c}`} + + {fmtMetric(pc.precision, true)} + + {fmtMetric(pc.recall, true)} + + {fmtMetric(pc.auc, false)} +
+
+ k-fold={cvResult.k} + {cvResult.k < cvResult.requestedK && + " (reduced — insufficient per-class samples)"} + {" · "}mean ± stdev across folds +
+
+ )} + + + + setSwipeOn(!!checked)} + style={{ marginTop: 12 }} + /> +
+ Drag the divider to compare pre-event imagery (left) with + post-event imagery (right). Satellite basemap is used on the pre + side when the layer has no pre-event imagery. +
+ + + + { + setUncertaintyOn(!!checked); + // Uncertainty view and Predicted view are mutually exclusive; + // switching this on drops the map back to the Labeled view. + if (checked) setViewMode("label"); + }} + /> +
+ Recolors every scored footprint by the model's predictive + uncertainty. Needs {MIN_PER_CLASS}+ labels in at least 2 classes. + A legend appears on the map. +
+
+ )} +
+ +
Click a building to label it · right-click to clear ·{" "} Ctrl+drag to box-label · 1/2/ 3 set class · P toggle view ·{" "} - Space show/hide footprints + Space show/hide footprints · with swipe on,{" "} + A/S/D snap divider left/center/right
)} diff --git a/ui/src/Components/InteractiveLabeler/interactiveModel.js b/ui/src/Components/InteractiveLabeler/interactiveModel.js index c9c6b3d..7f811d7 100644 --- a/ui/src/Components/InteractiveLabeler/interactiveModel.js +++ b/ui/src/Components/InteractiveLabeler/interactiveModel.js @@ -312,3 +312,196 @@ export function computeClassMetrics(preds, labels) { preds.filter((p, i) => p === labels[i]).length / (preds.length || 1); return { perClass, acc }; } + +// ── Rank-based binary AUC (Mann–Whitney U statistic) ──────────────────────── +// AUC = probability that a random positive scores above a random negative. +// Computed from ranks with average ranks for ties: +// AUC = (sum_of_ranks_of_positives - nPos*(nPos+1)/2) / (nPos*nNeg) +// Returns null for degenerate inputs (empty, or no positives / no negatives) +// since AUC is undefined without at least one of each class. +export function binaryAUC(scores, labels01) { + const n = scores.length; + if (n === 0 || labels01.length !== n) return null; + let nPos = 0; + let nNeg = 0; + for (let i = 0; i < n; i++) { + if (labels01[i] === 1) nPos++; + else nNeg++; + } + if (nPos === 0 || nNeg === 0) return null; + + // Rank scores ascending (1-based), assigning the average rank to ties. + const order = scores.map((_, i) => i).sort((a, b) => scores[a] - scores[b]); + const ranks = new Array(n); + let i = 0; + while (i < n) { + let j = i; + while (j + 1 < n && scores[order[j + 1]] === scores[order[i]]) j++; + const avgRank = (i + j) / 2 + 1; // midpoint of positions i..j, 1-based + for (let k = i; k <= j; k++) ranks[order[k]] = avgRank; + i = j + 1; + } + + let sumRanksPos = 0; + for (let r = 0; r < n; r++) { + if (labels01[r] === 1) sumRanksPos += ranks[r]; + } + return (sumRanksPos - (nPos * (nPos + 1)) / 2) / (nPos * nNeg); +} + +// Fisher–Yates in-place shuffle on a shallow copy (leaves the input intact). +function shuffle(arr) { + const a = [...arr]; + for (let i = a.length - 1; i > 0; i--) { + const j = Math.floor(Math.random() * (i + 1)); + [a[i], a[j]] = [a[j], a[i]]; + } + return a; +} + +// Mean + sample (n-1 denominator) standard deviation across `vals`. +// n >= 2 -> { mean, std, n } +// n == 1 -> { mean, std: 0, n } (a single fold has no spread) +// n == 0 -> { mean: null, std: null, n: 0 } +function meanStd(vals) { + const n = vals.length; + if (n === 0) return { mean: null, std: null, n: 0 }; + const mean = vals.reduce((s, v) => s + v, 0) / n; + if (n === 1) return { mean, std: 0, n: 1 }; + const variance = vals.reduce((s, v) => s + (v - mean) ** 2, 0) / (n - 1); + return { mean, std: Math.sqrt(variance), n }; +} + +// ── Stratified k-fold cross-validated per-class metrics ───────────────────── +// Runs stratified k-fold cross-validation over the labeled `entries` and +// reports per-class precision, recall, and one-vs-rest AUC, each aggregated +// across folds as mean ± sample standard deviation. +// +// Fold-stdev semantics: each fold produces (at most) one value per class per +// metric; we report the mean and the *sample* stdev (n-1 denominator) across +// the folds that yielded a defined value. This estimates the fold-to-fold +// variability of the metric — how sensitive it is to which subset is held +// out — NOT a confidence interval on any single trained model. Folds where a +// metric is undefined (precision when there are no predicted positives, recall +// when the fold has no true positives, or AUC when the held-out fold is +// single-class for that label) are skipped for that metric so one degenerate +// fold does not drag the estimate toward 0. +export async function crossValidateMetrics(entries, { k = 5, opts } = {}) { + const usable = (entries || []).filter((e) => isValidVector(e.features)); + const classes = [...new Set(usable.map((e) => e.label))].sort( + (a, b) => a - b + ); + if (classes.length < 2) { + return { + ok: false, + reason: "Need at least 2 labeled classes to run cross-validation.", + }; + } + + // Group members by class, count support, and find the smallest class. + const byClass = {}; + for (const c of classes) byClass[c] = []; + for (const e of usable) byClass[e.label].push(e); + const support = {}; + let minClassCount = Infinity; + for (const c of classes) { + support[c] = byClass[c].length; + if (support[c] < minClassCount) minClassCount = support[c]; + } + + // Effective folds cannot exceed the smallest class's member count, else a + // fold could be missing an entire class. + const effK = Math.min(k, minClassCount); + if (effK < 2) { + return { + ok: false, + reason: `Not enough samples per class for cross-validation (smallest class has ${minClassCount}; need at least 2).`, + }; + } + + const modelOpts = opts || { learningRate: 0.1, numSteps: 500, lambda: 0.01 }; + + // Stratified fold assignment: shuffle each class, then round-robin its + // members across the effK folds. Because every class has >= effK members, + // each fold receives at least one member of every class. + const folds = Array.from({ length: effK }, () => []); + for (const c of classes) { + shuffle(byClass[c]).forEach((e, i) => folds[i % effK].push(e)); + } + + const precVals = {}; + const recVals = {}; + const aucVals = {}; + for (const c of classes) { + precVals[c] = []; + recVals[c] = []; + aucVals[c] = []; + } + + for (let i = 0; i < effK; i++) { + const test = folds[i]; + const train = []; + for (let j = 0; j < effK; j++) if (j !== i) train.push(...folds[j]); + if (train.length === 0 || test.length === 0) continue; + // Guard against the (rare, tiny-data) case where the training split is + // missing a class — OvR needs both classes represented to be meaningful. + if (new Set(train.map((e) => e.label)).size < 2) continue; + + const ovr = new OvRLogisticRegression(modelOpts); + ovr.train( + train.map((e) => e.features), + train.map((e) => e.label) + ); + + const testX = test.map((e) => e.features); + const testTrue = test.map((e) => e.label); + const preds = ovr.predict(testX); + + for (const c of classes) { + // Per-class precision / recall from the argmax predictions. + let tp = 0; + let fp = 0; + let fn = 0; + for (let r = 0; r < testTrue.length; r++) { + const predC = preds[r] === c; + const trueC = testTrue[r] === c; + if (predC && trueC) tp++; + else if (predC && !trueC) fp++; + else if (!predC && trueC) fn++; + } + if (tp + fp > 0) precVals[c].push(tp / (tp + fp)); + if (tp + fn > 0) recVals[c].push(tp / (tp + fn)); + + // Per-class one-vs-rest AUC from the raw (un-normalized) sigmoid scores. + const clf = ovr.classifiers[c]; + if (clf) { + const y01 = testTrue.map((t) => (t === c ? 1 : 0)); + const auc = binaryAUC(clf.predictProba(testX), y01); + if (auc != null) aucVals[c].push(auc); + } + } + // Yield to the event loop between folds so a browser caller stays + // responsive (the map keeps repainting) instead of freezing for the whole + // CPU-bound cross-validation. + await new Promise((resolve) => setTimeout(resolve, 0)); + } + + const perClass = {}; + for (const c of classes) { + perClass[c] = { + precision: meanStd(precVals[c]), + recall: meanStd(recVals[c]), + auc: meanStd(aucVals[c]), + support: support[c], + }; + } + + return { + ok: true, + k: effK, + requestedK: k, + classes, + perClass, + nTotal: usable.length, + }; +}