diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 44635414..6d569f22 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -35,7 +35,7 @@ jobs: needs: test steps: - uses: actions/checkout@v7 - - uses: actions/setup-python@v6 + - uses: actions/setup-python@v7 with: python-version: ${{ env.PYTHON_VERSION }} cache: pip @@ -71,7 +71,7 @@ jobs: run: | sudo apt-get update sudo apt-get install -y libxcb-cursor0 libxcb-xinerama0 libxkbcommon-x11-0 libglib2.0-0 libegl1 libgl1-mesa-dev libdbus-1-3 xvfb libfuse2 imagemagick - - uses: actions/setup-python@v6 + - uses: actions/setup-python@v7 with: python-version: ${{ env.PYTHON_VERSION }} cache: pip @@ -122,7 +122,7 @@ jobs: needs: test steps: - uses: actions/checkout@v7 - - uses: actions/setup-python@v6 + - uses: actions/setup-python@v7 with: python-version: ${{ env.PYTHON_VERSION }} cache: pip @@ -164,7 +164,7 @@ jobs: needs: test steps: - uses: actions/checkout@v7 - - uses: actions/setup-python@v6 + - uses: actions/setup-python@v7 with: python-version: ${{ env.PYTHON_VERSION }} cache: pip diff --git a/.github/workflows/data-integrity.yml b/.github/workflows/data-integrity.yml index 9b325a87..2df9713a 100644 --- a/.github/workflows/data-integrity.yml +++ b/.github/workflows/data-integrity.yml @@ -17,7 +17,7 @@ jobs: issues: write steps: - uses: actions/checkout@v7 - - uses: actions/setup-python@v6 + - uses: actions/setup-python@v7 with: python-version: "3.12" cache: pip diff --git a/.github/workflows/quality.yml b/.github/workflows/quality.yml index fda8e971..040fa69d 100644 --- a/.github/workflows/quality.yml +++ b/.github/workflows/quality.yml @@ -10,7 +10,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v7 - - uses: actions/setup-python@v6 + - uses: actions/setup-python@v7 with: python-version: "3.12" cache: pip diff --git a/.github/workflows/screenshots.yml b/.github/workflows/screenshots.yml index 3bcd1d47..53aa228d 100644 --- a/.github/workflows/screenshots.yml +++ b/.github/workflows/screenshots.yml @@ -34,7 +34,7 @@ jobs: with: ref: ${{ inputs.branch || github.ref }} - - uses: actions/setup-python@v6 + - uses: actions/setup-python@v7 with: python-version: "3.12" cache: pip diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 7d55027e..7d4fb5ea 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -20,7 +20,7 @@ jobs: python-version: "3.12" steps: - uses: actions/checkout@v7 - - uses: actions/setup-python@v6 + - uses: actions/setup-python@v7 with: python-version: ${{ matrix.python-version }} cache: pip @@ -53,7 +53,7 @@ jobs: python-version: ["3.11", "3.12"] steps: - uses: actions/checkout@v7 - - uses: actions/setup-python@v6 + - uses: actions/setup-python@v7 with: python-version: ${{ matrix.python-version }} cache: pip diff --git a/CHANGELOG.md b/CHANGELOG.md index a48f1757..2ea0efbf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,30 +8,736 @@ Format follows [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). --- -## [1.15.1] — 2026-07-16 +## [1.16.0] — 2026-07-29 -> Hotfix release. Addresses a bug in the welcome wizard and Settings → -> Advanced's "move databases" flow reported shortly after 1.15.0 shipped. +> First stable release of the 1.16.0 cycle. Replaces the run of +> `1.16.0-beta.2` … `1.16.0-beta.16` builds — see git history for the +> detailed beta-by-beta log if needed. Headline of this cycle: a deep +> investigation into why exported caches weren't showing up correctly on +> Garmin handhelds ("Send to GPS"), plus a large-database performance +> pass and several data-integrity fixes for GSAK/GPX imports. + +### Added + +- **Garmin "Send to GPS" — full content and correct recognition (#656, + #453, #454, #455, #502)** — a multi-stage investigation, working + directly from GGZ/GPX byte comparisons against GSAK's own export of + the same caches, found and fixed the actual root cause: OpenSAK + exported GPX 1.1 with `groundspeak:cache` wrapped in an `` + element, while Garmin's on-device geocache parser is built around + GSAK's GPX 1.0 format, where `groundspeak:cache` is a direct child of + ``. Export format now matches GSAK's exactly. Along the way, + several content gaps were found and fixed: missing + `groundspeak:attributes`, `short_description`/`long_description`, and + `state`; logs hardcoded to the last 5 with text truncated at 500 + characters (now full history, no cap); and element order inside + `groundspeak:cache` corrected to match the official `cache.xsd` + sequence. Confirmed fixed by multiple testers across a GPSMAP 64s, + 66s, 66sr, and Montana 700 — description, hint, and previous logs all + display correctly now, and exported files are correctly recognized as + geocache files (not plain waypoints) on-device. +- **Custom Waypoints (Hotel/POI, Parking Area, Trailhead, etc.) get + proper icons, in both the app and on Garmin devices (#593, #660)** — + in-app, these now show their own distinct icon in the table, map, and + detail panel instead of a generic "unknown" icon. For Garmin export, + they're now written as plain GPX waypoints with a matching native + Garmin icon (e.g. "Parking Area", "Lodging", "Trail Head") instead of + showing up as an empty "fake geocache" with blank D/T stars. +- **"Use database name as filename" for GPS export** — new checkbox in + the Send to GPS dialog (community suggestion — GSAK has an + equivalent), on by default, pre-fills the export filename from the + active database's name for both file-mode and device-mode exports. + Its on/off state is remembered between exports. +- **Send to GPS collision handling** — device-mode exports ("Send to + GPS") now prompt before silently overwriting a same-named file on the + device, matching the protection file-mode export already had. The + "delete old files before upload" option now also covers GGZ exports + (previously GPX-only). +- **Large-database performance** (#627 and its follow-ups) — a new + lightweight query path (`apply_filters_lightweight()`/ + `apply_filters_auto()`) avoids ORM row-hydration cost with an + automatic, always-correct fallback to the full path whenever a filter + needs it; SQL pushdown extended to every remaining filter type; map + loading is dramatically faster via icon caching and bulk marker + loading; a "Max caches shown on map" setting and a "disable map + entirely" setting both target the same large-database load cost. + Measured on a 250,000-cache benchmark database: total time to show + all caches dropped by ~19% end-to-end, with individual steps (map + load, filtered queries) 2–48x faster depending on the scenario. A new + `scripts/benchmark_large_db.py` harness backs all of these numbers and + is available for future performance work too. +- **Default Column View** (#607) — named, saveable column configurations + (visible columns, widths, container/type display), with a toolbar + quick-switch dropdown and a designated global default for new/ + unconfigured databases, replacing the previous implicit "last used" + fallback. +- **Vertical gridlines in the cache table** (#463), and a **center point + picker for the distance filter** (#511) — choose Home, a saved home + point, the selected cache, or a manual coordinate as the filter's + center, plus a matching right-click "Set as center point" action. + +### Fixed + +- **Cache distance not calculated after adding/editing a waypoint** + (#662) — a newly added cache (including Custom Waypoints) or one with + edited coordinates showed no distance and sorted to the bottom of the + list until the center/home point was switched away and back. Distance + and bearing are now recalculated immediately in both cases. +- **Wrong attribute mappings from GSAK import** (#615) — 42 of 70 + Groundspeak attribute IDs were mapped to the wrong attribute, also + silently affecting the attribute filter regardless of import source. +- **Corrected-coordinate caches lost their original coordinates on GSAK + database import** (#614) — the original (pre-solve) position is now + read from GSAK's `Corrected` table instead of the already-corrected + `Latitude`/`Longitude` columns. +- **Hidden date (and log dates) lost when importing Project-GC-style + GPX** (#617) — an explicit UTC-offset timestamp format + (`+00:00`) wasn't recognized by the old parser and silently came back + as `None`. +- **Logs wiped on every re-import instead of accumulating** (#618) — a + partial GPX/PQ re-import now merges logs (update-in-place for a + matching ID, add new ones, keep existing ones not present in the + current file) instead of deleting and rebuilding from that file alone. +- **Community Celebration Event caches imported as generic "Event + Cache"** (#591) — narrow name-based fallback for this un-typed, + time-limited Groundspeak program. +- **New database didn't inherit column settings** (#606) — falls back to + the last-used configuration instead of hard-coded factory defaults. +- **Redundant distance recalculation on every startup** (#579) — skipped + when nothing about the database or home point has changed since the + last run. +- **Boolean filters bypassed their SQL indexes** (#628) — `IS true`/ + `IS false` changed to the index-usable `= true`/`= false` form. +- **Hint markup incorrectly ROT13-scrambled** (#595) — bracketed markup + like `[br]` is now left untouched by the hint cipher. +- Several dark-theme/UI fixes: unreadable placeholder text (#624), an + unreadable Where-filter SQL error box (#613), and Country/Region/ + County columns left-aligned instead of centered (#603). +- Database dropdown/lists not sorted alphabetically (#531, #601), and a + misleading "database created" confirmation message (#464). + +--- + +## [1.16.0-beta.16] — 2026-07-28 + +> **Beta release** — further "Send to GPS" polish following tester +> feedback on beta.15, plus proper Garmin icons for Custom Waypoints +> and Adventure Lab caches. + +### Fixed + +- **"Send to GPS" silently overwrote a same-named file on the device + with no warning (#656)** — reported by CheminerWill after testing + beta.15. The existing #501 fix only covered file-mode exports; + device-mode ("Send to GPS") now runs the same collision check: if a + file with the same name already exists in the device's Garmin/GPX or + Garmin/GGZ folder, you're prompted for a new name (with an + auto-suggested next-available name), unless "delete old files before + upload" is checked, in which case the old files are cleared first. + That checkbox now also works for GGZ exports (previously GPX-only). +- **Custom Waypoints (Hotel/POI, Parking Area, Trailhead, etc.) showed + up on Garmin devices as empty "fake geocaches"** — with blank D/T + stars and `Size: (Not Chosen)` — instead of looking like the simple + waypoints they are. They're now exported as plain GPX waypoints with + a proper native Garmin icon (e.g. "Parking Area", "Lodging", + "Trail Head"). Confirmed fixed on a GPSMAP 64s via GPX import. + +### Added + +- **"Use database name as filename"** checkbox in the GPS export + dialog (community suggestion — GSAK has an equivalent), checked by + default, covering both file-mode and device-mode exports. Pre-fills + the export filename with the currently active database's name + instead of a fixed "opensak" default, reducing how often the + collision prompt above gets triggered in normal day-to-day use. + +### Notes + +- Adventure Lab stages were also given a distinct icon attempt + ("Flag, Blue" instead of the standard geocache icon) — this had no + visible effect on the tested GPSMAP 64s (device firmware appears to + always use its own icon for anything containing a full + `groundspeak:cache` block, regardless of the `sym` field). Left in + place since it's harmless and may help on other device/firmware + combinations; Lab stages keep their full description/D-T/hint + content either way, which was the more important fix. + +--- + +## [1.16.0-beta.15] — 2026-07-28 + +> **Beta release** — a cluster of fixes to GPX/GGZ export for Garmin +> devices ("Send to GPS"), including the root cause of Garmin firmware +> not recognizing exported files as geocaches at all. ### Fixed -- **Database move fails with "No such file or directory" on a fresh install - (#609)** — on a brand-new installation, the auto-created "Default" - database entry only exists as an in-memory/metadata record until the app - actually opens it; no physical `.db` file exists yet at that point. If - the user chose a separate database folder during the welcome wizard and - asked to move existing databases, this looked like an existing database - ("You have 1 existing database(s)") but the move itself then failed, - since there was no file to copy. Database entries without a physical - file on disk are now recognized as such — they're no longer counted in - the "existing databases" prompt, and moving them simply updates the - recorded path instead of failing. -- **Database/install folder pickers showed forward slashes on Windows - (#609)** — the folder chosen via the "Browse…" button in the welcome - wizard and Settings → Advanced is now normalized to the platform's - native path separator for display, instead of showing Qt's internal - forward-slash path verbatim (e.g. `E:/Users/...` instead of - `E:\Users\...`). +- **GPX/GGZ export missing attributes, descriptions, state, and full log + history (#656)** — `generate_gpx()` (used by both GPX and GGZ export, + since GGZ embeds a generated GPX internally) never wrote + `groundspeak:attributes`, `groundspeak:short_description` / + `long_description`, or `groundspeak:state` at all, and hardcoded log + export to only the last 5 logs with text truncated to 500 characters. + Confirmed via a controlled side-by-side comparison against GSAK's + export of the same cache (GC.com direct download and GSAK both + included full attributes, description, and complete log history for + the same cache; OpenSAK's export was missing all of it). All of the + above is now exported in full, with no artificial limits. + +- **Garmin devices not recognizing OpenSAK's GPX/GGZ exports as geocaches + (#656)** — the underlying root cause of the above and of related + reports (#453, #454, #455, #502): OpenSAK exported GPX 1.1 + (`xmlns=".../GPX/1/1"`) with `groundspeak:cache` wrapped inside a + GPX-1.1-style `` element. GSAK exports GPX 1.0 + (`xmlns=".../GPX/1/0"`) with `groundspeak:cache` as a *direct child* of + `` — no `` wrapper — which is what Garmin's on-device + geocache parser is built around. Export format now matches GSAK's + exactly, including declaring the `groundspeak` XML namespace locally + on the `` element itself rather than at the GPX + root. Confirmed fixed on a Garmin GPSMAP 64s: description, logs and + hint all now display correctly for an exported cache, where previously + only the hint displayed and description/logs did not. + +--- + +## [1.16.0-beta.14] — 2026-07-26 + +> **Beta release** — vertical gridlines in the database grid (#463). + +### Added + +- **Vertical gridlines in the cache table** (#463) — columns in the main + cache grid now have a thin vertical separator line at each column + boundary, in addition to the existing alternating row colours. The + line colour follows the active theme's palette (light/dark), so no + additional theme handling is needed. User-configurable colours, + independent show/hide toggles for horizontal vs. vertical lines, and + the proposed new Appearance settings tab remain out of scope for now — + basic functionality comes first, per the discussion on the issue. + +--- + +## [1.16.0-beta.13] — 2026-07-26 + +> **Beta release** — a new Default Column View system (named, saveable +> column configurations with a toolbar quick-switch, replacing #606's +> implicit "last used" fallback), plus three small UI fixes: centered +> text columns, dark-theme placeholder text, and a readable Where-filter +> error box in dark mode. + +### Fixed + +- **Country/Region/County columns left-aligned instead of centered (#603)** + — #431 centered "similar short-value columns" (Placed By, dates, etc.) + but missed Country, Region (state) and County. Centered here for the + same consistency #431 was going for. +- **Placeholder/hint text invisible in Dark theme (#624)** — `QPalette`'s + `PlaceholderText` role wasn't set explicitly for either theme, so Qt + fell back to a derived default that was unreadable in dark mode. Both + palettes now set it explicitly to a legible, dimmed gray. +- **Where-filter SQL error box unreadable in Windows dark mode (#613)** — + the error box used a hardcoded light-theme style (dark-red text on a + transparent background); in dark mode that rendered as dark-red-on-dark + gray. It now picks an explicit, theme-appropriate style via the + existing `effective_theme()` helper. + +### Added + +- **Default Column View (#607)** — the "Choose columns" dialog now supports + named, saveable "Column Views" (visible columns, widths, container/type + display), parallel to saved filter profiles. A saved view can be picked + from a dropdown, and one view can be marked as the global default via a + new "Set as Default" button (shown with a ★ in the dropdown). Any + database without its own explicit column configuration — including + brand-new databases — now falls back to the designated default view + instead of the hard-coded factory defaults. This replaces #606's + implicit "last used" fallback, which silently changed on every save + regardless of user intent; setting the default is now an explicit, + deliberate action and does not retroactively affect databases that + already have their own saved configuration. A quick-switch Column View + dropdown has also been added to the main toolbar, next to the existing + filter-profile dropdown, so a saved view can be applied to the active + database with one click, without opening the dialog. + +--- + +## [1.16.0-beta.12] — 2026-07-26 + +> **Beta release** — a cluster of GPX-import fixes reported by the +> community: a lost hidden date from Project-GC exports, logs being wiped +> on every re-import instead of accumulating, Community Celebration Event +> caches showing the wrong type, and column settings not carrying over to +> a new database. + +### Fixed + +- **Hidden date lost when importing a GPX file from Project-GC** (#617) — + `_parse_datetime()` only handled a bare or `Z`-suffixed ISO 8601 + timestamp (geocaching.com's own format). Project-GC instead exports an + explicit UTC offset (`2026-03-16T00:00:00+00:00`), which matched none + of the old `strptime` patterns and silently came back as `None`, + dropping the hidden date entirely. Now tries `datetime.fromisoformat()` + first and converts properly to UTC (a `+02:00` offset is converted to + the correct UTC instant, not just relabelled), falling back to the + previous patterns for anything else. The same function also parses log + dates, so this fixes any log-date loss from Project-GC-style exports + too, not just hidden dates. + +- **Logs removed from one GPX import to another** (#618) — every + re-import (e.g. loading a new Pocket Query) deleted *all* of a cache's + existing logs and rebuilt them from that file alone. Since a single + GPX/PQ typically only carries a cache's most recent handful of logs, + this meant older logs not present in that particular file were + permanently lost on the next import — unlike GSAK, which lets logs + accumulate over time. Logs are now merged instead: a matching log ID is + updated in place (e.g. an edited log text), a new one is added, and any + existing log absent from the current file is left untouched. + `log_count`, `last_log_date`, and `dnf_date` are now derived from the + full merged set rather than just the current file's logs, so a partial + re-import can no longer move them backwards or silently clear them. + +- **Community Celebration Event caches imported as generic "Event + Cache"** (#591) — geocaching.com's own machine-readable + `` field has no distinct value for Community + Celebration Events (a limited-run program, May 2020 – Dec 2021) — it's + always exported as plain "Event Cache". The actual event type only + survives in the free-text cache name (e.g. "Karlínská kasárna - + Community Celebration Event"). Added a narrow fallback: when + `groundspeak:type` is exactly "Event Cache" and the cache name contains + the literal phrase "Community Celebration Event", the cache is now + classified as such — using the dedicated type/icon that already existed + but was never being reached. Verified against a real-world GPX export + for the reported cache (GC8T83E); a plain Event Cache, or another type + merely mentioning the phrase in its name, is correctly left unchanged. + +- **Displayed field when importing .gpx not consistent with previous + settings** (#606) — column visibility and widths are saved per database + name, so a brand-new database had no saved key yet and fell straight + back to the hard-coded defaults, silently reverting any customisation + (e.g. an added Country/State/County column) already made in another + database. Added a global "last used" fallback key, updated on every + save, that a database with no settings of its own now falls back to + before the hard-coded defaults — so a new database inherits whatever + was last configured, anywhere. + +--- + +## [1.16.0-beta.11] — 2026-07-22 + +> **Beta release** — a setting to disable the map panel entirely (and a +> visible "disabled" placeholder instead of an empty-looking map), plus a +> setting to cap the map to the nearest N caches from your home +> coordinate. Both target the same thing #627 already identified: map +> load is the largest remaining cost in "show me my caches" on a large +> database. + +### Added + +- **Limit map to nearest N caches from home coordinate** (#639) — + combines @nagisml's "max caches shown on the map" suggestion (on #638) + with sorting by distance from the active home coordinate, since a map + with hundreds of thousands of pins isn't very readable at normal zoom + anyway. New "Max caches shown on map" spinbox in the Map settings tab + (0 = unlimited), default 2000. The map now gets its **own** fetch, + independent of the table's — same active filterset, but sorted by + distance and limited — so the table's own result set and sort order + are completely unaffected either way. + + Added `push_limit` to `apply_filters_lightweight()`/`apply_filters_auto()` + (default `False`, no behavior change for existing callers): when the + whole filterset is SQL-pushed *and* the sort field is SQL-sortable, the + limit is pushed into the SQL query itself (`LIMIT` after `ORDER BY`) + instead of fetching every filtered row and slicing in Python. Measured + directly (100,000-cache database, distance-sorted, no filter): the + existing Python-slice `limit` took ~3.0s regardless of requested size + (500 through 5000 all fetch and construct every row before slicing); + pushing a real SQL `LIMIT` took 0.31-0.56s, correctly scaling with the + requested size. Falls back to the existing (always-correct) Python-slice + behavior whenever those two conditions aren't both met — a SQL `LIMIT` + applied before a Python-only sort or Python-only filter pass would + silently return the wrong N rows, so this is deliberately conservative; + 13 dedicated tests cover both the cases where it activates and the ones + where it correctly must not. + + End-to-end confirmed (100,000-cache database, fetch + JSON payload + build): 4.65s (unlimited) → 0.46s (default limit of 2000) — ~10x + faster, with the JSON payload itself shrinking from ~195MB to ~3.9MB. + +- **Setting to disable the map panel** (#638) — new "Map" tab in the + Settings dialog (split out from General, so future map-specific + settings have a natural shared home) with a "Show map" checkbox, + defaulting to on (opt-out, zero behavior change for anyone who doesn't + touch it). When off, all three `mainwindow.py` refresh paths skip + building and loading the map's marker data entirely — since #627, + map load has been the single largest remaining cost in "show me my + caches" on a large database, bigger than the database query itself. + Measured directly: the map-load step (Python-side payload build) drops + from ~1.8s to effectively 0s on a 100,000-cache database when disabled. + Toggling the setting back on mid-session needs no special handling — + the existing Settings-dialog-close flow already calls + `_refresh_cache_list()` unconditionally, which now correctly re-populates + the map on the next call since the guard re-checks the setting fresh + every time. Table contents and sort order are completely unaffected + either way. + + **Follow-up:** the map's own base tiles/zoom controls rendered + regardless of the setting — only the cache markers were actually + skipped — which looked like a stuck or empty map rather than an + intentional off state (reported after testing the initial version). + A `QStackedWidget` now swaps in a plain "Map disabled" placeholder page + instead, so the off state is visually unambiguous. Also skips the + map's own page reload (`reload_map()`) while disabled, since there's + no point refreshing tiles nobody's looking at — it reloads normally + the next time the setting is re-enabled. Translated to all 8 languages. + +--- + +## [1.16.0-beta.10] — 2026-07-22 + +> **Beta release** — SQL pushdown for the last group of filters that had +> none at all, following up on #627's lightweight query path. + +### Added + +- **SQL pushdown for remaining scalar-column filters** (#633) — + `UserFlagFilter`, `LockedFilter`, `DnfFilter`, `FtfFilter`, + `FavoritePointsFilter`, `HasCorrectedFilter`/`NoCorrectedFilter`, + `FoundByMeDateFilter`, `DnfDateFilter`, and `LastLogDateFilter` + previously had no `apply_to_query()` at all, always falling back to a + full Python `matches()` scan. Under the ORM path this barely mattered + (#631 found the Python pass was only ~2% of `apply_filters()`'s time — + ORM hydration dominated regardless of whether a filter narrowed the SQL + query or not). Under the lightweight query path (beta.9) the picture is + different: without SQL pushdown, `apply_filters_lightweight()` must + still construct a `LightweightCache` for every single row before + Python-filtering it down, so a highly selective filter with no pushdown + costs almost as much as fetching the whole table. Measured directly on + a 100,000-cache database: `FtfFilter` (0.6% selectivity) went from + 2.42s to 0.05s (~48x faster); `UserFlagFilter` (~5%) from 2.49s to + 0.16s (~15x); `DnfFilter` (~7%) from 2.39s to 0.20s (~12x). + + Each `apply_to_query()` mirrors its `matches()` counterpart exactly, + including NULL handling — `FoundByMeDateFilter`/`DnfDateFilter` treat a + NULL date as "include" (found/DNF but undated), while + `LastLogDateFilter` treats NULL as "exclude", and both are preserved + precisely in SQL. `HasCorrectedFilter`/`NoCorrectedFilter` use a + correlated `EXISTS`/`NOT EXISTS` against `user_notes`, which needed an + explicit `.correlate(Cache)` — without it, `apply_filters_lightweight()` + raised `InvalidRequestError` because its `select()` already outerjoins + `user_notes` for corrected-coordinate display, confusing SQLAlchemy's + auto-correlation. Only broke on the lightweight path, not the full ORM + path — caught by testing both, not just one. + + 25 new parity tests (including NULL edge cases for every field + involved) confirm every filter's SQL and Python forms agree exactly; + full unit-test suite (2136 tests) green, mypy clean. + +--- + +## [1.16.0-beta.9] — 2026-07-22 + +> **Beta release** — the lightweight query path (#627): large databases +> load dramatically faster in both the cache table and the map, on top of +> #628-#631's smaller fixes from the last two betas. This is a +> default-behavior change for every install, not opt-in — see below for +> why that's safe. + +### Added + +- **Lightweight query path** (#627) — `apply_filters_lightweight()`, a new + function in `filters/engine.py` alongside `apply_filters()`, fetches + cache rows via a SQLAlchemy Core `select()` instead of + `session.query(Cache)`, avoiding the ORM row-hydration cost already + identified as `apply_filters()`'s dominant expense (#631). Results come + back as `LightweightCache` objects — duck-typed to expose the same + attribute names as a real `Cache` for every column the table and map + actually use — with an automatic, transparent fallback to the existing + `apply_filters()` ORM path whenever a filter needs a relationship or one + of the three heavy/deferred text fields (`short_description`, + `long_description`, `encoded_hints`). That fallback means this is always + correct, never returning wrong or incomplete results — only sometimes + slower than it could be. + + `mainwindow.py`'s table and map refresh now go through a single + `apply_filters_auto()` entry point that always attempts the lightweight + path. A thorough compatibility audit (`CacheTableModel`'s every column, + sort key, and tooltip; `map_widget.py`'s `_do_load_caches()`, + `_effective_coords()`, and pin-icon generation) found **zero** source + changes were needed in either consumer — both already only touch scalar + fields, cached count columns, or `.user_note`'s three attributes, never + a relationship collection directly. Row selection already reloads a + full `Cache` via the established `_load_full_cache(gc_code)` pattern + regardless of what's currently in the table. + + Confirmed final numbers (250,000-cache database, via the real + `apply_filters_auto()` wiring): + + | Scenario | `apply_filters` | `apply_filters_auto` | Speedup | + |---|---|---|---| + | No filter | 8.05s | 3.10s | ~2.6x faster | + | Exclude archived | 11.02s | 3.81s | ~2.9x faster | + | Within 50km | 1.52s | 0.97s | ~1.6x faster | + | `CacheTableModel.load()` | 0.198s | 0.027s | ~7.3x faster | + + **Two real bugs were found and fixed during testing, before release:** + a `LightweightCache` design that eagerly copied every one of its ~52 + fields at construction time fixed a table-load regression (delegating + every attribute through `__getattr__` was costing more than the + fetch-side win it was meant to complement) but overcorrected, nearly + erasing the fetch-side win in the process — the final design only + eagerly copies the three fields `CacheTableModel` touches + unconditionally on every row (`id`, `distance`, `bearing`), leaving + everything else lazy. Separately, `reload_caches_full()` — the helper + GPX/LOC/GGZ export, KML export, GPS-device export, and the trip planner + all use to reload full cache data before generating output — checked + `isinstance(c, Cache)`, which silently excluded every `LightweightCache` + row from its reload and would have crashed all four export paths the + moment they touched a deferred field; fixed by recognizing both types as + reloadable. Both were caught by the project's own test suite (the + second one by the e2e suite specifically) before ever reaching a tagged + release. + + Confidence for shipping this as default (not opt-in) behavior comes + from: the lightweight path's own automatic per-filterset fallback to the + exact same full-ORM code path used today; full parity test coverage + (`test_filter_sql_parity.py`, `test_filter_lightweight.py`) proving + `apply_filters_lightweight()` never diverges from `apply_filters()`'s + result set across every filter type, NULL edge case, and composition; + dedicated compatibility audits and test suites for both the table and + the map with zero source changes needed in either; and a full pass of + the unit suite (2111 tests), the e2e suite (244 tests), and a + 250,000-cache benchmark, all green. + +- **Benchmark harness measures the lightweight query path** (#628) — + `scripts/benchmark_large_db.py` now also runs its three `apply_filters` + scenarios through `apply_filters_auto()`, and runs the map/table-load + steps against both result sets, so a single report shows the full + before/after picture instead of requiring a separate isolated A/B + script. Fixed a measurement-fairness bug found while adding this: the + icon HTML `@lru_cache` (#629) meant whichever "Map load" measurement ran + first in the script paid the one-time cache-warming cost and the second + one benefited "for free" — fixed with an explicit warmup pass before + either timed measurement. + +--- + +## [1.16.0-beta.8] — 2026-07-22 + +> **Beta release** — a small, safe correctness fix in the filter engine's +> SQL pushdown, spun off from the #627 large-database investigation. + +### Fixed + +- **Boolean filters silently bypassed their indexes** (#628, part of #627) + — `FoundFilter`, `ArchivedFilter`, `AvailableFilter`, `AvailabilityFilter`, + `PremiumFilter`, and `NonPremiumFilter` used `Cache..is_(True)` / + `.is_(False)` in their SQL pushdown, which compiles to ` IS true` / + `IS false`. SQLite's query planner cannot use an index for that form — + verified directly against SQLite 3.45 with `EXPLAIN QUERY PLAN` — even + though the functionally identical ` = true` / `= false` (what + `== True`/`== False` compiles to) is index-usable and the relevant + indexes have existed since #214. `.is_(None)` (NULL checks, e.g. + `DifficultyFilter`'s unknown-difficulty handling) was never affected and + is unchanged. Real-world impact is small at current database sizes — + isolated A/B testing showed ~0.11s either way for a selective filter on + 100,000 caches, since raw SQL execution is dwarfed by ORM row hydration + (same finding as #631) — but this restores the indexing intent from + #214 at zero cost and zero risk. + +--- + +## [1.16.0-beta.7] — 2026-07-22 + +> **Beta release** — large-database performance work (see #627): map load +> is dramatically faster on big databases thanks to icon caching and bulk +> marker loading, plus a small, safe win in the filter engine. Includes a +> new benchmark harness so every step here — and future ones — can be +> measured instead of guessed at. +> +> Measured on a 250,000-cache synthetic benchmark database +> (`scripts/benchmark_large_db.py`): map load dropped from 11.34s to 4.45s +> (-61%), total time to show all caches dropped from 37.58s to 30.42s +> (-19%). Full before/after table in #628. + +### Added + +- **Large-database benchmark harness** (#628, part of #627) — + `scripts/benchmark_large_db.py` generates a synthetic database at a + configurable scale (default 250,000 caches) and measures distance + recalculation, `apply_filters()`, map load, table load, and info-bar + update, printing a table (optionally markdown) for pasting into GitHub + issues. Every performance change in #627 is now measured against this + harness rather than eyeballed. + +### Improved + +- **Cache map pin HTML generation** (#629, part of #627) — `get_map_pin_html()` + now caches its output with `@lru_cache(maxsize=256)`. The HTML (including + base64-encoded SVG) only depends on `(cache_type, found, dnf)`, a small + bounded set of combinations, but was previously rebuilt from scratch for + every visible cache on every map load. On a 100,000-cache benchmark + database (see #628's `scripts/benchmark_large_db.py`), map load time + dropped from ~10.1s to ~3.2s (~68%). + +- **Bulk-load map markers with chunked clustering** (#630, part of #627) — + the map's `loadCaches()` called Leaflet.markercluster's `addLayer()` once + per cache, which rebuilds the library's spatial index on every single + call. It now builds all markers first and adds them in one + `addLayers()` bulk call, with `chunkedLoading: true` so the browser's UI + thread stays responsive while a large marker set loads. The post-load + pan/fit-bounds step is deferred until every chunk has actually been + added (via `chunkProgress`), so it still reflects the complete marker + set instead of a partially-loaded one. + +- **Skip redundant Python filter pass when fully SQL-pushed** (#631, part + of #627) — `apply_filters()` now skips its Python-level + `filterset.matches()` re-scan when every filter in the filterset was + already pushed into the SQL `WHERE` clause, since every row `query.all()` + returns already satisfies it. Measured impact is modest — the Python pass + itself is only ~2% of `apply_filters()`'s time even on a 100,000-cache + database with a large result set (~6.8s total, ~0.13s of which was the + redundant pass); ORM hydration dominates and is unaffected by this + change. Still a safe, zero-cost win, and it required introducing a new + `BaseFilter.sql_exact` flag: while implementing this, testing surfaced + that `DistanceFilter`'s SQL pushdown is a bounding-box *pre-narrowing* + only (not an exact translation — it ignores `min_km` entirely and + doesn't have the true circle shape), so the naive "non-None + `apply_to_query()` == fully handled" assumption would have silently + dropped the `min_km` check for distance-filtered results. `sql_exact` + lets a filter opt out of counting toward the skip decision while still + contributing its SQL pre-narrowing; `DistanceFilter` is the only filter + that needs it. + +--- + +## [1.16.0-beta.6] — 2026-07-21 + +> **Beta release** — two data-integrity fixes for GSAK-database imports: +> attribute names and the attribute filter were often wrong, and corrected +> (solved-puzzle) caches lost their original coordinates on import. + +### Fixed + +- **Wrong attribute settings from GSAK database import** (#615) — 42 of the + 70 Groundspeak attribute IDs in OpenSAK's internal attribute table were + mapped to the wrong attribute (e.g. id 31 resolved to "Food nearby" + instead of "Camping available"). Beyond GSAK-database imports, this also + affected the attribute filter, which built its checkbox labels and + underlying filter values from the same table — so filtering by attribute + could silently return the wrong caches regardless of import source. + Rebuilt and verified against real GPX exports from geocaching.com. + +- **Caches with corrected coordinates lose the original coordinates when + importing GSAK database** (#614) — GSAK's own `Latitude`/`Longitude` + columns reflect the *corrected* position once a cache has been solved, + not the original/posted coordinates. OpenSAK imported these directly as + the cache's primary position, silently discarding the true original + location on every GSAK-database import of a solved cache. The original + position is now read from GSAK's `Corrected` table instead. + +--- + +## [1.16.0-beta.5] — 2026-07-16 + +> **Beta release** — startup no longer recalculates every cache's distance +> unnecessarily, which should noticeably speed up launch on large databases. + +### Fixed + +- **Redundant distance recalculation on every startup** (#579) — the app + recalculated distance/bearing for every cache on every launch, even + though nothing about the database or home point had changed since the + last session. On large databases (100k+ caches) this made startup + noticeably slow with no visual indication of what was happening. + `recalculate_distances()` now persists the centre point and distance + method it was run with, and on startup the app checks this — plus a + cheap single-row spot-check against the database — before deciding + whether a full recalculation is actually needed. Normal startup now + skips it entirely; a database synced from another machine with a + different home point (or otherwise modified outside this OpenSAK + install) still triggers a full recalculation as before. + +--- + +## [1.16.0-beta.4] — 2026-07-15 + +> **Beta release** — the database list/dropdown is now alphabetically +> sorted, plus a small message cleanup. + +### Fixed + +- **Database list/dropdown was not sorted alphabetically** (#531, #601) — + the toolbar database dropdown, the Manage Databases dialog, and the + database picker in Move Caches, GSAK import, and GPX/PQ import all + listed databases in the order they were added/imported instead of + alphabetically. All of these now show databases sorted alphabetically + (case-insensitive) by name, matching GSAK's behaviour. +- **"Database created" message told the user to manually activate it** + (#464) — creating a new database already switches to it automatically, + but the confirmation dialog still said to click "Switch to this" to + activate it. The message now simply confirms the database was created + and is active. + +--- + +## [1.16.0-beta.3] — 2026-07-15 + +> **Beta release** — pick any cache, saved home point, or coordinate as the +> distance filter's center (#511), plus two small bugfixes. + +### Added + +- **Center point picker for the distance filter** (#511) — the "Afstand" + filter no longer always centers on Home. Choose Home, any saved home + point, the currently selected cache, or a manually entered coordinate as + the center, and set an optional minimum distance alongside the existing + maximum (both were already supported by the filter engine; only the + maximum was previously exposed in the dialog). Built as a standalone, + reusable widget for future reuse (planned for #558). +- **"Set as center point" (right-click)** (#511) — right-click any cache or + custom waypoint (e.g. a hotel added via Waypoint → Custom Waypoint) and + choose "Sæt som centerpunkt" to recompute the Distance column for every + cache from that point, exactly like switching Home. The chosen point's + GC code/name is shown in the info bar's "Centerpunkt" field and in the + Home dropdown until you pick a saved home point or another cache. + +### Fixed + +- **Hint markup was being ROT13-scrambled** (#595) — geocaching.com's own + hint markup (`[br]` for a line break, place-name tags like `[Étape]` in + French hints) was incorrectly rotated along with the rest of the hint + text, so `[br]` showed up as its ROT13'd form `[oe]` instead of a line + break. Bracketed markup is now left untouched by the ROT13 + encode/decode, and `[br]` renders as an actual line break in both the + cache detail hint tab and KML export. +- **Website: corrected GSAK's freeware date** (#589) — the landing page's + comparison table said GSAK became freeware in 2021; per research from a + long-time GSAK user (French GSAK user since 2011), the free v9.0.0 + shipped in 2019, with the last forum-provided patch dating from 2022. + +--- + +## [1.16.0-beta.2] — 2026-07-15 + +> **Beta release** — custom waypoint types get their own icons, and the +> found-smiley icon set is simplified (#593). + +### Added + +- **Custom waypoint types now have their own icons** — Parking Area, + Trailhead, Stage, Final Location, Reference Point, Waypoint, Hotel/POI + and Custom each get a distinct icon in the table, map and detail panel, + instead of all sharing the generic "unknown" (?) icon. Overridable via + the same `icons/cache_types/` user-icon mechanism as #519. + +### Changed + +- **Simplified the found-smiley icon set** (#593) — removed the 12 unused + colour variants and the per-type colour-selection code behind them. + Only `gold` (Found overlay + "Found" column) and `dark_blue` (DNF + overlay) were ever actually shown in the app; the rest was dead + code/assets. Reported by a community member in the OpenSAK Facebook + group. --- diff --git a/scripts/benchmark_large_db.py b/scripts/benchmark_large_db.py new file mode 100644 index 00000000..a5944cce --- /dev/null +++ b/scripts/benchmark_large_db.py @@ -0,0 +1,629 @@ +#!/usr/bin/env python3 +""" +scripts/benchmark_large_db.py — Large-database performance benchmark harness. + +Part of #628 (large-database performance meta-issue #627). + +Why this exists: @nagisml's manual benchmark on #579 showed that the +distance-recalculation fix only accounts for a small slice of total load +time on a large database — apply_filters() (~9.5s) and map load (~8.6s) +dominate. Every follow-up optimization in #627 (icon caching, map bulk +loading, skipped redundant filtering, the lightweight query path) needs to +be measured against the same baseline, on the same synthetic data, or we're +guessing instead of measuring. + +This script: + 1. Generates a synthetic OpenSAK database at a configurable scale (default + 250,000 caches, each with a random number of logs/attributes/ + trackables, scattered both near and far from a home point so + distance-filtering scenarios are meaningful). + 2. Measures the same steps as @nagisml's benchmark comment, PLUS the + lightweight query path added in #627 beta.9-11: + - distance recalc (cold, full) + - distance spot-check (warm — confirms #579's skip path is taken) + - distance recalc (invalidated — home point changed, fallback path) + - DB query / apply_filters, for three scenarios (None, exclude + archived, distance-filtered) + - the same three scenarios again via apply_filters_auto() — what + mainwindow.py actually calls for every table/map refresh since + beta.10/11. Always attempts the lightweight query path (beta.9), + falling back to the apply_filters() path above automatically for + anything that needs a relationship or deferred text field. + - map load (Python-side payload build — JSON + pin-icon generation), + once against apply_filters()'s result and once against + apply_filters_auto()'s, so the report shows both paths directly + - table load (CacheTableModel), same before/after pairing + - info-bar update + 3. Prints a table in the same format as the #579 benchmark comment, so + results can be pasted directly into issue comments for before/after + comparisons. + +Safety: this script NEVER touches your real OpenSAK settings or databases. +It isolates the settings store to a throwaway temp directory and only ever +opens the synthetic database file you point it at. + +Usage: + source .venv/bin/activate + python scripts/benchmark_large_db.py + python scripts/benchmark_large_db.py --cache-count 250000 --keep + python scripts/benchmark_large_db.py --db-path /tmp/bench.sqlite --skip-generate + +Steps 5 and 6 (map load, table load) need PySide6 with +QT_QPA_PLATFORM=offscreen; if that's not available they're skipped with a +warning and the rest of the benchmark still runs: + + QT_QPA_PLATFORM=offscreen python scripts/benchmark_large_db.py + +Note on scope: the map-load step measures the real Python-side production +code path (get_map_pin_html() via map_widget._cache_pin_html(), JSON +building, template-literal escaping) — exactly what #629 and the Python +side of #630 touch. It does NOT measure actual Leaflet/JS marker-clustering +time inside the browser (that's #630's chunkedLoading/addLayers() work) — +QtWebEngine's JS execution isn't something this headless script can time +synchronously. Verify that part manually in the running app. +""" + +from __future__ import annotations + +import argparse +import random +import sys +import tempfile +import time +from dataclasses import dataclass +from pathlib import Path +from typing import Callable, TypeVar + +REPO_ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(REPO_ROOT / "src")) + +from sqlalchemy import text # noqa: E402 + +# ── Synthetic data pools ───────────────────────────────────────────────────── + +HOME_LAT, HOME_LON = 55.6761, 12.5683 # Copenhagen — matches settings.py default + +CACHE_TYPES = [ + "Traditional Cache", "Multi-cache", "Unknown Cache", "Earthcache", + "Letterbox Hybrid", "Wherigo Cache", "Virtual Cache", "Event Cache", +] +CONTAINERS = ["Nano", "Micro", "Small", "Regular", "Large", "Not chosen", "Other"] +RATINGS = [1.0, 1.5, 2.0, 2.5, 3.0, 3.5, 4.0, 4.5, 5.0] +LOG_TYPES = ["Found it", "Didn't find it", "Write note", "Owner Maintenance"] +COUNTRIES = ["Denmark", "Sweden", "Germany", "Norway", "Netherlands"] +ATTRIBUTE_POOL = [ + (1, "Dogs allowed"), (2, "Bicycles"), (4, "Kids friendly"), + (7, "Wheelchair accessible"), (13, "Available 24-7"), (24, "Night cache"), + (32, "Poison plants"), (43, "Field puzzle"), +] + + +@dataclass +class GenConfig: + cache_count: int + seed: int + near_fraction: float = 0.7 # fraction of caches scattered near the home point + near_radius_km: float = 100.0 + logs_low_max: int = 10 # most caches: 0-10 logs + logs_low_prob: float = 0.70 + logs_mid_max: int = 40 # some caches: 10-40 logs + logs_mid_prob: float = 0.25 + logs_high_max: int = 150 # a few "power" caches: 40-150 logs + chunk_size: int = 5000 + + +def _rand_latlon_near(rng: random.Random, lat0: float, lon0: float, radius_km: float) -> tuple[float, float]: + """Uniform-ish random point within *radius_km* of (lat0, lon0).""" + import math + r = radius_km * math.sqrt(rng.random()) / 111.0 # ~111 km per degree latitude + theta = rng.random() * 2 * math.pi + dlat = r * math.cos(theta) + dlon = r * math.sin(theta) / max(math.cos(math.radians(lat0)), 0.01) + return lat0 + dlat, lon0 + dlon + + +def _rand_latlon_anywhere(rng: random.Random) -> tuple[float, float]: + return rng.uniform(-60.0, 70.0), rng.uniform(-170.0, 170.0) + + +def _rand_log_count(rng: random.Random, cfg: GenConfig) -> int: + roll = rng.random() + if roll < cfg.logs_low_prob: + return rng.randint(0, cfg.logs_low_max) + elif roll < cfg.logs_low_prob + cfg.logs_mid_prob: + return rng.randint(cfg.logs_low_max, cfg.logs_mid_max) + return rng.randint(cfg.logs_mid_max, cfg.logs_high_max) + + +def generate_database(db_path: Path, cfg: GenConfig) -> None: + """Build a synthetic OpenSAK database at *db_path* with *cfg.cache_count* caches.""" + from datetime import datetime, timedelta, timezone + + from opensak.db.database import init_db + from opensak.db.models import Attribute, Cache, Log, Trackable + + if db_path.exists(): + db_path.unlink() + for suffix in ("-shm", "-wal"): + side = Path(str(db_path) + suffix) + if side.exists(): + side.unlink() + + engine = init_db(db_path) + rng = random.Random(cfg.seed) + + with engine.begin() as conn: + # Throwaway synthetic DB — full durability is not needed while generating. + conn.execute(text("PRAGMA synchronous=OFF")) + conn.execute(text("PRAGMA cache_size=-131072")) # ~128 MB page cache + + cache_rows: list[dict] = [] + log_rows: list[dict] = [] + attr_rows: list[dict] = [] + tb_rows: list[dict] = [] + + log_id_seq = 1 + now = datetime.now(timezone.utc) + + from sqlalchemy import insert as _sa_insert + + def _flush(conn) -> None: + nonlocal cache_rows, log_rows, attr_rows, tb_rows + if cache_rows: + conn.execute(_sa_insert(Cache), cache_rows) + cache_rows = [] + if log_rows: + conn.execute(_sa_insert(Log), log_rows) + log_rows = [] + if attr_rows: + conn.execute(_sa_insert(Attribute), attr_rows) + attr_rows = [] + if tb_rows: + conn.execute(_sa_insert(Trackable), tb_rows) + tb_rows = [] + + with engine.begin() as conn: + for i in range(1, cfg.cache_count + 1): + cache_id = i + near = rng.random() < cfg.near_fraction + lat, lon = ( + _rand_latlon_near(rng, HOME_LAT, HOME_LON, cfg.near_radius_km) + if near else _rand_latlon_anywhere(rng) + ) + archived = rng.random() < 0.05 + available = archived or rng.random() > 0.03 + found = rng.random() < 0.30 + n_logs = _rand_log_count(rng, cfg) + + cache_rows.append(dict( + id=cache_id, + gc_code=f"GC{cache_id:06X}", + name=f"Benchmark Cache {cache_id}", + cache_type=rng.choice(CACHE_TYPES), + container=rng.choice(CONTAINERS), + latitude=lat, + longitude=lon, + difficulty=rng.choice(RATINGS), + terrain=rng.choice(RATINGS), + placed_by=f"Owner{cache_id % 500}", + owner_name=f"Owner{cache_id % 500}", + owner_id=str(cache_id % 500), + hidden_date=now - timedelta(days=rng.randint(30, 4000)), + last_updated=now, + available=available, + archived=archived, + premium_only=rng.random() < 0.05, + short_description="A synthetic benchmark cache.", + short_desc_html=False, + long_description="Placeholder long description text for benchmarking.", + long_desc_html=False, + encoded_hints=None, + country=rng.choice(COUNTRIES), + state=None, + county=None, + found=found, + found_date=(now - timedelta(days=rng.randint(0, 1000))) if found else None, + dnf=(not found) and rng.random() < 0.1, + dnf_date=None, + first_to_find=found and rng.random() < 0.02, + user_flag=rng.random() < 0.05, + user_sort=None, + user_data_1=None, user_data_2=None, user_data_3=None, user_data_4=None, + distance=None, + bearing=None, + log_count=n_logs, + trackable_count=0, + found_log_count=1 if found else 0, + waypoint_count=0, + last_log_date=(now - timedelta(days=rng.randint(0, 60))) if n_logs else None, + source_file="benchmark_large_db.py", + locked=False, + )) + + for j in range(n_logs): + log_rows.append(dict( + id=log_id_seq, + cache_id=cache_id, + log_id=f"bench_{cache_id}_{j}", + log_type=rng.choice(LOG_TYPES), + log_date=now - timedelta(days=rng.randint(0, 2000)), + finder=f"Finder{rng.randint(0, 5000)}", + finder_id=str(rng.randint(0, 5000)), + text="Great cache, thanks for the hide!", + text_encoded=False, + latitude=None, + longitude=None, + logged_by_owner=False, + )) + log_id_seq += 1 + + for attr_id, attr_name in rng.sample(ATTRIBUTE_POOL, k=rng.randint(0, 4)): + attr_rows.append(dict( + cache_id=cache_id, + attribute_id=attr_id, + name=attr_name, + is_on=rng.random() < 0.8, + )) + + if rng.random() < 0.10: + tb_rows.append(dict( + cache_id=cache_id, + ref=f"TB{cache_id}A", + name=f"Travel Bug {cache_id}", + )) + + if i % cfg.chunk_size == 0: + _flush(conn) + _flush(conn) + + with engine.begin() as conn: + conn.execute(text("PRAGMA synchronous=FULL")) + conn.execute(text("PRAGMA cache_size=-2000")) + conn.execute(text("ANALYZE")) + + +# ── Settings isolation (never touch the real user's config) ───────────────── + +def _isolate_settings(tmp_dir: Path) -> None: + from opensak import settings_store as ss + fresh = ss.SettingsStore() + fresh._data = {} + fresh._path = tmp_dir / "opensak.json" + ss._store = fresh + + import opensak.gui.settings as smod + smod._settings = None + + import opensak.db.manager as mgr + mgr._manager = None + + +# ── Benchmark steps ─────────────────────────────────────────────────────────── + +@dataclass +class StepResult: + label: str + seconds: float | None + detail: str = "" + + +_T = TypeVar("_T") + + +def _timed(label: str, fn: Callable[..., _T], *args, **kwargs) -> tuple[StepResult, _T]: + t0 = time.perf_counter() + result = fn(*args, **kwargs) + dt = time.perf_counter() - t0 + return StepResult(label, dt), result + + +def _warm_icon_cache() -> None: + """Pre-warm get_map_pin_html()'s @lru_cache (#629) before any timed map + load runs. + + Without this, whichever "Map load" measurement happens to run FIRST in + the script pays the one-time cold-cache cost (SVG loading + base64 + encoding for every distinct cache_type/found/dnf combination) and every + later map-load measurement in the same process benefits "for free" — + making a before/after comparison between two Map load steps an artifact + of run order, not of the actual difference between apply_filters() and + apply_filters_auto(). Warming the cache once up front, before either + measurement, keeps that comparison fair: both start from a warm cache, + same as the real app after its first-ever map render. + """ + try: + from opensak.gui.map_widget import _cache_pin_html + except ImportError: + return + cache_types = [ + "Traditional Cache", "Multi-cache", "Unknown Cache", "Earthcache", + "Letterbox Hybrid", "Wherigo Cache", "Virtual Cache", "Event Cache", + ] + for ct in cache_types: + for found in (True, False): + for dnf in (True, False): + _cache_pin_html(ct, found, dnf) + + +def bench_distance_recalc() -> list[StepResult]: + from opensak.db.database import distances_up_to_date, recalculate_distances + + results = [] + + r, n = _timed("Distance recalc (cold, full)", recalculate_distances, HOME_LAT, HOME_LON) + r.detail = f"{n} caches updated" + results.append(r) + + r, up_to_date = _timed("Distance spot-check (warm)", distances_up_to_date, HOME_LAT, HOME_LON) + r.detail = "up to date, skipped" if up_to_date else "WARNING: expected up-to-date" + results.append(r) + + # Simulate a database synced from elsewhere with a different home point — + # distances_up_to_date() should report False, forcing the fallback path. + other_lat, other_lon = HOME_LAT + 5.0, HOME_LON + 5.0 + r, invalidated = _timed("Distance spot-check (invalidated)", distances_up_to_date, other_lat, other_lon) + r.detail = "correctly detected stale" if not invalidated else "WARNING: expected stale" + results.append(r) + + r2, n2 = _timed("Distance recalc (after invalidation, full)", recalculate_distances, other_lat, other_lon) + r2.detail = f"{n2} caches updated" + results.append(r2) + + # Restore the original home point's persisted values for later steps. + recalculate_distances(HOME_LAT, HOME_LON) + + return results + + +def bench_apply_filters() -> tuple[list[StepResult], list]: + from opensak.db.database import get_session + from opensak.filters.engine import ArchivedFilter, DistanceFilter, FilterSet, WhereClauseFilter, apply_filters + + results = [] + + with get_session() as session: + r, caches = _timed("apply_filters — no filter", apply_filters, session, None, None) + r.detail = f"{len(caches)} caches" + results.append(r) + + with get_session() as session: + fs = FilterSet(mode="AND") + fs.add(WhereClauseFilter("archived = 0")) + r, caches = _timed("apply_filters — exclude archived", apply_filters, session, fs, None) + r.detail = f"{len(caches)} caches" + results.append(r) + + with get_session() as session: + fs = FilterSet(mode="AND") + fs.add(DistanceFilter(HOME_LAT, HOME_LON, max_km=50.0)) + r, caches = _timed("apply_filters — within 50km", apply_filters, session, fs, None) + r.detail = f"{len(caches)} caches" + results.append(r) + + return results, caches # last (smallest) result set reused by later steps + + +def bench_apply_filters_auto() -> tuple[list[StepResult], list]: + """Same three scenarios as bench_apply_filters(), but via + apply_filters_auto() — what mainwindow.py actually calls for every + table/map refresh since #627 beta.10/11. apply_filters_auto() always + attempts the lightweight query path (#627 beta.9), automatically + falling back to the same full-ORM apply_filters() path above for any + filterset that needs a relationship or deferred text field (none of + these three do). Run alongside bench_apply_filters() so a single + report shows both the old and the now-current path directly, instead + of requiring a separate isolated A/B script. + """ + from opensak.db.database import get_session + from opensak.filters.engine import ArchivedFilter, DistanceFilter, FilterSet, WhereClauseFilter, apply_filters_auto + + results = [] + + with get_session() as session: + r, caches = _timed("apply_filters_auto — no filter", apply_filters_auto, session, None, None) + r.detail = f"{len(caches)} caches" + results.append(r) + + with get_session() as session: + fs = FilterSet(mode="AND") + fs.add(WhereClauseFilter("archived = 0")) + r, caches = _timed("apply_filters_auto — exclude archived", apply_filters_auto, session, fs, None) + r.detail = f"{len(caches)} caches" + results.append(r) + + with get_session() as session: + fs = FilterSet(mode="AND") + fs.add(DistanceFilter(HOME_LAT, HOME_LON, max_km=50.0)) + r, caches = _timed("apply_filters_auto — within 50km", apply_filters_auto, session, fs, None) + r.detail = f"{len(caches)} caches" + results.append(r) + + return results, caches # last (smallest) result set reused by later steps + + +def bench_map_load(caches: list, label_suffix: str = "") -> list[StepResult]: + """Measure the Python-side map payload build (JSON + pin icons). + + Reuses the exact production functions map_widget._do_load_caches() calls, + without needing a live QWebEngineView. Does NOT measure JS-side Leaflet + clustering time — see module docstring. + """ + label = f"Map load (Python-side payload){label_suffix}" + try: + import json as _json + + from opensak.gps.garmin import _effective_coords + from opensak.gui.map_widget import _cache_pin_html + except ImportError as exc: + return [StepResult(label, None, f"skipped — {exc}")] + + def _build_payload(caches): + data = [] + for c in caches: + if c.latitude is None or c.longitude is None: + continue + note = getattr(c, "user_note", None) + has_corrected = bool(note and getattr(note, "is_corrected", False)) + eff_lat, eff_lon = _effective_coords(c) + data.append({ + "gc_code": c.gc_code, "name": c.name or "", "cache_type": c.cache_type or "", + "difficulty": c.difficulty or 0, "terrain": c.terrain or 0, + "lat": c.latitude, "lon": c.longitude, "clat": eff_lat, "clon": eff_lon, + "corrected": has_corrected, "corrected_label": "Corrected", + "pin_html": _cache_pin_html(c.cache_type or "", bool(c.found), bool(c.dnf)), + "found": c.found, + }) + json_str = _json.dumps(data, ensure_ascii=False) + json_str = json_str.replace("\\", "\\\\").replace("`", "\\`") + return json_str + + r, payload = _timed(label, _build_payload, caches) + r.detail = f"{len(caches)} caches, {len(payload) / 1024:.0f} KB JSON" + return [r] + + +def bench_table_load(caches: list, label_suffix: str = "") -> list[StepResult]: + label = f"Table load (CacheTableModel){label_suffix}" + try: + import os + os.environ.setdefault("QT_QPA_PLATFORM", "offscreen") + from PySide6.QtWidgets import QApplication + + from opensak.gui.cache_table import CacheTableModel + except ImportError as exc: + return [StepResult(label, None, f"skipped — {exc}")] + + app = QApplication.instance() or QApplication([]) + model = CacheTableModel() + r, _ = _timed(label, model.load, caches) + r.detail = f"{len(caches)} caches" + return [r] + + +def bench_info_bar(caches: list) -> list[StepResult]: + """Replicates mainwindow._update_info_bar()'s per-cache aggregate cost + without needing a full MainWindow/GUI instance.""" + from opensak.db.database import get_session + from opensak.db.models import Cache + + def _run() -> tuple[int, int, int, int]: + with get_session() as session: + total_in_db = session.query(Cache).count() + found = sum(max(c.found_log_count, 1) for c in caches if c.found) + flagged = sum(1 for c in caches if c.user_flag) + inactive = sum(1 for c in caches if c.archived or not c.available) + return total_in_db, found, flagged, inactive + + r, (total_in_db, found, flagged, inactive) = _timed("Info-bar update", _run) + r.detail = f"{total_in_db} in DB, {found} found, {flagged} flagged, {inactive} inactive" + return [r] + + +# ── Reporting ───────────────────────────────────────────────────────────────── + +def print_results(all_results: list[StepResult]) -> None: + label_w = max(len(r.label) for r in all_results) + 2 + print() + print(f"{'Step':<{label_w}}{'Time':>10} Detail") + print("-" * (label_w + 10 + 3 + 40)) + total = 0.0 + for r in all_results: + time_str = f"{r.seconds:.4f}s" if r.seconds is not None else "—" + print(f"{r.label:<{label_w}}{time_str:>10} {r.detail}") + if r.seconds is not None: + total += r.seconds + print("-" * (label_w + 10 + 3 + 40)) + print(f"{'Total':<{label_w}}{total:.2f}s") + print() + + +def print_markdown_table(all_results: list[StepResult]) -> None: + print("| Step | Time | Detail |") + print("|---|---|---|") + total = 0.0 + for r in all_results: + time_str = f"{r.seconds:.4f}s" if r.seconds is not None else "—" + print(f"| {r.label} | {time_str} | {r.detail} |") + if r.seconds is not None: + total += r.seconds + print(f"| **Total** | **{total:.2f}s** | |") + + +# ── Main ────────────────────────────────────────────────────────────────────── + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--cache-count", type=int, default=250_000, help="Number of synthetic caches to generate (default: 250000)") + parser.add_argument("--db-path", type=Path, default=None, help="Path for the synthetic database (default: a temp file)") + parser.add_argument("--seed", type=int, default=42, help="Random seed for reproducible generation") + parser.add_argument("--keep", action="store_true", help="Don't delete the generated database afterwards") + parser.add_argument("--skip-generate", action="store_true", help="Reuse an existing database at --db-path instead of generating a new one") + parser.add_argument("--markdown", action="store_true", help="Also print results as a markdown table (for pasting into GitHub issues)") + args = parser.parse_args() + + db_path = args.db_path or Path(tempfile.gettempdir()) / "opensak_benchmark.sqlite" + tmp_settings_dir = Path(tempfile.mkdtemp(prefix="opensak_benchmark_settings_")) + _isolate_settings(tmp_settings_dir) + + if not args.skip_generate: + print(f"Generating {args.cache_count:,} synthetic caches at {db_path} (seed={args.seed})...") + t0 = time.perf_counter() + generate_database(db_path, GenConfig(cache_count=args.cache_count, seed=args.seed)) + print(f" done in {time.perf_counter() - t0:.1f}s") + else: + from opensak.db.database import init_db + init_db(db_path) + print(f"Reusing existing database at {db_path}") + + from opensak.gui.settings import get_settings + s = get_settings() + s.home_lat = HOME_LAT + s.home_lon = HOME_LON + + all_results: list[StepResult] = [] + all_results += bench_distance_recalc() + + filter_results, smallest_caches = bench_apply_filters() + all_results += filter_results + + auto_filter_results, _ = bench_apply_filters_auto() + all_results += auto_filter_results + + # map/table/info-bar steps run against the largest (unfiltered) result set, + # matching @nagisml's "Total to caches shown" methodology. + from opensak.db.database import get_session + from opensak.filters.engine import apply_filters, apply_filters_auto + with get_session() as session: + all_caches = apply_filters(session, None, None) + + _warm_icon_cache() # keep both Map load measurements below a fair A/B — see docstring + all_results += bench_map_load(all_caches) + all_results += bench_table_load(all_caches) + all_results += bench_info_bar(all_caches) + + # #627 beta.9-11: apply_filters_auto() (what mainwindow.py actually + # calls for every table/map refresh) is the now-current path — rerun + # map/table load against its result set too, so the report shows the + # full realistic before/after picture in one run instead of requiring + # a separate isolated A/B script. + with get_session() as session: + all_caches_auto = apply_filters_auto(session, None, None) + + all_results += bench_map_load(all_caches_auto, label_suffix=" — apply_filters_auto") + all_results += bench_table_load(all_caches_auto, label_suffix=" — apply_filters_auto") + + print_results(all_results) + if args.markdown: + print_markdown_table(all_results) + + if not args.keep and not args.skip_generate: + db_path.unlink(missing_ok=True) + for suffix in ("-shm", "-wal"): + Path(str(db_path) + suffix).unlink(missing_ok=True) + print(f"(deleted {db_path} — pass --keep to retain it)") + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/site/assets/screenshots/column-chooser.png b/site/assets/screenshots/column-chooser.png index e1f9c6ef..c10f8bec 100644 Binary files a/site/assets/screenshots/column-chooser.png and b/site/assets/screenshots/column-chooser.png differ diff --git a/site/assets/screenshots/filter-dialog-attributes.png b/site/assets/screenshots/filter-dialog-attributes.png index e4f1bc02..07a05795 100644 Binary files a/site/assets/screenshots/filter-dialog-attributes.png and b/site/assets/screenshots/filter-dialog-attributes.png differ diff --git a/site/assets/screenshots/filter-dialog-general.png b/site/assets/screenshots/filter-dialog-general.png index c53e7b3e..7e8a5ded 100644 Binary files a/site/assets/screenshots/filter-dialog-general.png and b/site/assets/screenshots/filter-dialog-general.png differ diff --git a/site/assets/screenshots/gps-export.png b/site/assets/screenshots/gps-export.png index 88350c58..dc3e1f3c 100644 Binary files a/site/assets/screenshots/gps-export.png and b/site/assets/screenshots/gps-export.png differ diff --git a/site/assets/screenshots/main-window.png b/site/assets/screenshots/main-window.png index fbd4743c..6894493b 100644 Binary files a/site/assets/screenshots/main-window.png and b/site/assets/screenshots/main-window.png differ diff --git a/site/assets/screenshots/settings-general.png b/site/assets/screenshots/settings-general.png index 67d245d1..1aff643e 100644 Binary files a/site/assets/screenshots/settings-general.png and b/site/assets/screenshots/settings-general.png differ diff --git a/site/index.html b/site/index.html index 4dcf2d8d..faf1578a 100644 --- a/site/index.html +++ b/site/index.html @@ -726,7 +726,7 @@

How does OpenSAK compare?

-

* GSAK became freeware in 2021 after active development ceased.

+

* GSAK became freeware with version 9.0.0 in 2019; active development effectively ceased in 2022.

diff --git a/site/user-guide.html b/site/user-guide.html index 4e153143..c483ed15 100644 --- a/site/user-guide.html +++ b/site/user-guide.html @@ -3,7 +3,7 @@ -OpenSAK User Guide — v1.15.1 +OpenSAK User Guide — v1.16.0