Release 2026.1

.github/workflows/ci-deploy.yaml

@@ -50,6 +50,9 @@ jobs:
           submodules: recursive
       - name: Run sccache-cache
         uses: mozilla-actions/sccache-action@v0.0.9
+      - name: Install libuuid
+        run: |
+          yum -y install libuuid-devel
       - name: Configure
         run: |
           python3 -m venv venv && . ./venv/bin/activate

.github/workflows/coverage.yml

@@ -20,7 +20,8 @@ jobs:
       uses: mozilla-actions/sccache-action@v0.0.9
     - name: Install dependencies
       run: |
-        sudo apt-get install ninja-build
+        sudo apt-get update
+        sudo apt-get install -y ninja-build uuid-dev
         pip install gcovr
         gcovr --version
     - name: Configure

CMakeLists.txt

@@ -7,12 +7,16 @@ project(mapget LANGUAGES CXX C)
 # Allow version to be set from command line for CI/CD
 # For local development, use the default version
 if(NOT DEFINED MAPGET_VERSION)
-  set(MAPGET_VERSION 2025.5.1)
+  set(MAPGET_VERSION 2026.1.0)
 endif()
 
 set(CMAKE_CXX_STANDARD 20)
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
 
+if (WIN32)
+  add_compile_definitions(NOMINMAX)
+endif()
+
 include(FetchContent)
 include(GNUInstallDirs)
 
@@ -22,12 +26,21 @@ if (CMAKE_CURRENT_SOURCE_DIR STREQUAL CMAKE_SOURCE_DIR)
   set(MAPGET_WITH_HTTPLIB ON CACHE BOOL "Enable mapget-http-datasource and mapget-http-service libraries.")
   set(MAPGET_ENABLE_TESTING ON CACHE BOOL "Enable testing.")
   set(MAPGET_BUILD_EXAMPLES ON CACHE BOOL "Build examples.")
+  # Prevent CPM dependencies (e.g. Drogon/Trantor) from registering their own
+  # tests into this project's ctest run.
+  set(BUILD_TESTING OFF CACHE BOOL "Disable dependency tests" FORCE)
 endif()
 
 option(MAPGET_WITH_WHEEL "Enable mapget Python wheel (output to WHEEL_DEPLOY_DIRECTORY).")
 option(MAPGET_WITH_SERVICE "Enable mapget-service library. Requires threads.")
 option(MAPGET_WITH_HTTPLIB "Enable mapget-http-datasource and mapget-http-service libraries.")
 
+if (MAPGET_ENABLE_TESTING)
+  # Enable testing before adding CPM dependencies so stale/third-party CTest
+  # files don't linger in the build tree.
+  enable_testing()
+endif()
+
 set(Python3_FIND_STRATEGY LOCATION)
 
 if (NOT MSVC)
@@ -112,7 +125,6 @@ endif()
 # tests
 
 if (MAPGET_ENABLE_TESTING)
-  enable_testing()
   add_subdirectory(test/unit)
 
   if (MAPGET_WITH_WHEEL)

cmake/deps.cmake

@@ -14,17 +14,53 @@ CPMAddPackage(
     OPTIONS
         "EXPECTED_BUILD_TESTS OFF"
         "EXPECTED_BUILD_PACKAGE_DEB OFF")
-CPMAddPackage(
-    URI "gh:Klebert-Engineering/simfil@0.6.2"
-    OPTIONS
-        "SIMFIL_WITH_MODEL_JSON ON"
-        "SIMFIL_SHARED OFF")
+
+set(MAPGET_SIMFIL_SOURCE_DIR "" CACHE PATH
+    "Local simfil source directory to use instead of fetching from Git.")
+
+set(_mapget_simfil_source_dir "${MAPGET_SIMFIL_SOURCE_DIR}")
+if ("${_mapget_simfil_source_dir}" STREQUAL ""
+    AND EXISTS "${CMAKE_CURRENT_LIST_DIR}/../../simfil/CMakeLists.txt")
+    set(_mapget_simfil_source_dir "${CMAKE_CURRENT_LIST_DIR}/../../simfil")
+endif()
+
+if (NOT "${_mapget_simfil_source_dir}" STREQUAL "")
+    message(STATUS "Using local simfil from ${_mapget_simfil_source_dir}")
+    CPMAddPackage(
+        NAME simfil
+        SOURCE_DIR "${_mapget_simfil_source_dir}"
+        OPTIONS
+            "SIMFIL_WITH_MODEL_JSON ON"
+            "SIMFIL_SHARED OFF")
+else()
+    CPMAddPackage(
+        URI "gh:Klebert-Engineering/simfil#v0.7.0"
+        OPTIONS
+            "SIMFIL_WITH_MODEL_JSON ON"
+            "SIMFIL_SHARED OFF")
+endif()
+
 CPMAddPackage(
     URI "gl:eidheim/tiny-process-library#8bbb5a"  # Switch to release > 2.0.4 once available
     OPTIONS
         "BUILD_TESTING OFF")
 
 if (MAPGET_WITH_WHEEL OR MAPGET_WITH_HTTPLIB OR MAPGET_ENABLE_TESTING)
+    # OpenSSL's Configure script needs a "full" Perl distribution. Git for
+    # Windows ships a minimal perl that is missing required modules (e.g.
+    # Locale::Maketext::Simple), causing OpenSSL builds to fail.
+    if (WIN32)
+        if (NOT DEFINED PERL_EXECUTABLE OR PERL_EXECUTABLE MATCHES "[\\\\/]Git[\\\\/]usr[\\\\/]bin[\\\\/]perl\\.exe$")
+            find_program(_MAPGET_STRAWBERRY_PERL
+                NAMES perl.exe
+                PATHS "C:/Strawberry/perl/bin"
+                NO_DEFAULT_PATH)
+            if (_MAPGET_STRAWBERRY_PERL)
+                set(PERL_EXECUTABLE "${_MAPGET_STRAWBERRY_PERL}" CACHE FILEPATH "" FORCE)
+            endif()
+        endif()
+    endif()
+
     set (OPENSSL_VERSION openssl-3.5.2)
     CPMAddPackage("gh:klebert-engineering/openssl-cmake@1.0.0")
     CPMAddPackage(
@@ -40,19 +76,67 @@ if (MAPGET_WITH_WHEEL OR MAPGET_WITH_HTTPLIB OR MAPGET_ENABLE_TESTING)
     endif()
 
     CPMAddPackage(
-        URI "gh:yhirose/cpp-httplib@0.15.3"
+        NAME jsoncpp
+        GIT_REPOSITORY https://github.com/open-source-parsers/jsoncpp
+        GIT_TAG 1.9.5
+        GIT_SHALLOW ON
+        OPTIONS
+            "JSONCPP_WITH_TESTS OFF"
+            "JSONCPP_WITH_POST_BUILD_UNITTEST OFF"
+            "JSONCPP_WITH_PKGCONFIG_SUPPORT OFF"
+            "JSONCPP_WITH_CMAKE_PACKAGE OFF"
+            "BUILD_SHARED_LIBS OFF"
+            "BUILD_STATIC_LIBS ON"
+            "BUILD_OBJECT_LIBS OFF")
+    # Help Drogon's FindJsoncpp.cmake locate jsoncpp when built via CPM.
+    set(JSONCPP_INCLUDE_DIRS "${jsoncpp_SOURCE_DIR}/include" CACHE PATH "" FORCE)
+    set(JSONCPP_LIBRARIES jsoncpp_static CACHE STRING "" FORCE)
+    # CPM generates a dummy package redirect config at
+    # `${CMAKE_FIND_PACKAGE_REDIRECTS_DIR}/jsoncpp-config.cmake`. Drogon uses
+    # `find_package(Jsoncpp)` (config-first), so make that redirect actually
+    # define the expected `Jsoncpp_lib` target.
+    if (DEFINED CMAKE_FIND_PACKAGE_REDIRECTS_DIR)
+        file(MAKE_DIRECTORY "${CMAKE_FIND_PACKAGE_REDIRECTS_DIR}")
+        file(WRITE "${CMAKE_FIND_PACKAGE_REDIRECTS_DIR}/jsoncpp-extra.cmake" [=[
+if(NOT TARGET Jsoncpp_lib)
+  add_library(Jsoncpp_lib INTERFACE)
+  target_include_directories(Jsoncpp_lib INTERFACE "${JSONCPP_INCLUDE_DIRS}")
+  target_link_libraries(Jsoncpp_lib INTERFACE ${JSONCPP_LIBRARIES})
+endif()
+]=])
+    endif()
+
+    # Drogon defines install(EXPORT ...) rules unconditionally, which fail when
+    # used as a subproject with CPM-provided dependencies (zlib/jsoncpp/etc).
+    # Since mapget only needs Drogon for building, temporarily suppress install
+    # rule generation while configuring Drogon.
+    set(_MAPGET_PREV_SKIP_INSTALL_RULES "${CMAKE_SKIP_INSTALL_RULES}")
+    if (DEFINED BUILD_TESTING)
+        set(_MAPGET_PREV_BUILD_TESTING "${BUILD_TESTING}")
+    endif()
+    set(CMAKE_SKIP_INSTALL_RULES ON)
+    set(BUILD_TESTING OFF)
+
+    CPMAddPackage(
+        URI "gh:drogonframework/drogon@1.9.7"
         OPTIONS
-            "CPPHTTPLIB_USE_POLL ON"
-            "HTTPLIB_USE_CERTS_FROM_MACOSX_KEYCHAIN OFF"
-            "HTTPLIB_INSTALL OFF"
-            "HTTPLIB_USE_OPENSSL_IF_AVAILABLE OFF"
-            "HTTPLIB_USE_ZLIB_IF_AVAILABLE OFF")
-    # Manually enable openssl/zlib in httplib to avoid FindPackage calls.
-    target_compile_definitions(httplib INTERFACE
-        CPPHTTPLIB_OPENSSL_SUPPORT
-        CPPHTTPLIB_ZLIB_SUPPORT)
-    target_link_libraries(httplib INTERFACE
-        OpenSSL::SSL OpenSSL::Crypto ZLIB::ZLIB)
+            "BUILD_CTL OFF"
+            "BUILD_EXAMPLES OFF"
+            "BUILD_ORM OFF"
+            "BUILD_BROTLI OFF"
+            "BUILD_YAML_CONFIG OFF"
+            "BUILD_SHARED_LIBS OFF"
+            "USE_SUBMODULE ON"
+            "USE_STATIC_LIBS_ONLY OFF"
+            "USE_POSTGRESQL OFF"
+            "USE_MYSQL OFF"
+            "USE_SQLITE3 OFF"
+        GIT_SUBMODULES "trantor")
+
+    set(CMAKE_SKIP_INSTALL_RULES "${_MAPGET_PREV_SKIP_INSTALL_RULES}")
+    if (DEFINED _MAPGET_PREV_BUILD_TESTING)
+        set(BUILD_TESTING "${_MAPGET_PREV_BUILD_TESTING}")
+    endif()
 
     CPMAddPackage(
         URI "gh:jbeder/yaml-cpp#aa8d4e@0.8.0" # Use > 0.8.0 once available.
@@ -64,6 +148,7 @@ if (MAPGET_WITH_WHEEL OR MAPGET_WITH_HTTPLIB OR MAPGET_ENABLE_TESTING)
     CPMAddPackage("gh:CLIUtils/CLI11@2.5.0")
     CPMAddPackage("gh:pboettch/json-schema-validator#2.3.0")
     CPMAddPackage("gh:okdshin/PicoSHA2@1.0.1")
+
 endif ()
 
 if (MAPGET_WITH_WHEEL AND NOT TARGET pybind11)
@@ -76,7 +161,7 @@ if (MAPGET_WITH_SERVICE OR MAPGET_WITH_HTTPLIB OR MAPGET_ENABLE_TESTING)
 endif()
 
 if (MAPGET_WITH_WHEEL AND NOT TARGET python-cmake-wheel)
-    CPMAddPackage("gh:Klebert-Engineering/python-cmake-wheel@1.1.0")
+    CPMAddPackage("gh:Klebert-Engineering/python-cmake-wheel#v1.2.7@1.2.7")
 endif()
 
 if (MAPGET_ENABLE_TESTING)

docs/mapget-api.md

@@ -1,12 +1,12 @@
-# REST/GeoJSON API Guide
+# HTTP / WebSocket API Guide
 
-Mapget exposes a small HTTP API that lets clients discover datasources, stream tiles, abort long‑running requests, locate features by ID and inspect or update the running configuration. This guide describes the endpoints and their request and response formats.
+Mapget exposes a small HTTP + WebSocket API that lets clients discover datasources, stream tiles, locate features by ID and inspect or update the running configuration. Interactive tile streaming now uses a WebSocket control channel plus `/tiles/next` pull requests for the binary tile data. This guide describes the endpoints and their request and response formats.
 
 ## Base URL and formats
 
 The server started by `mapget serve` listens on the configured host and port (by default on all interfaces and an automatically chosen port). All endpoints are rooted at that host and port.
 
-Requests that send JSON use `Content-Type: application/json`. Tile streaming supports two response encodings, selected via the `Accept` header:
+Requests that send JSON use `Content-Type: application/json`. HTTP tile streaming supports two response encodings, selected via the `Accept` header:
 
 - `Accept: application/jsonl` returns a JSON‑Lines stream where each line is one JSON object.
 - `Accept: application/binary` returns a compact binary stream optimized for high-volume traffic.
@@ -21,28 +21,61 @@ The binary format and the logical feature model are described in more detail in
 - **Request body:** none
 - **Response:** `application/json` array of datasource descriptors
 
-Each item contains map ID, available layers and basic metadata. This endpoint is typically used by frontends to discover which maps and layers can be requested via `/tiles`.
+Each item contains map ID, available layers and basic metadata. Each layer entry includes its type, `zoomLevels`, `coverage`, staged-loading metadata (`stages`, optional `stageLabels`, `highFidelityStage`) and feature-type information. This endpoint is typically used by frontends to discover which maps and layers can be requested via `/tiles`.
 
-## `/tiles` – stream tiles
+## `/tiles` – stream tiles (HTTP)
 
-`POST /tiles` streams tiles for one or more map–layer combinations. It is the main data retrieval endpoint used by clients such as erdblick.
+`POST /tiles` streams tiles for one or more map–layer combinations.
 
 - **Method:** `POST`
 - **Request body (JSON):**
   - `requests`: array of objects, each with:
     - `mapId`: string, ID of the map to query.
     - `layerId`: string, ID of the layer within that map.
-    - `tileIds`: array of numeric tile IDs in mapget’s tiling scheme.
+    - either `tileIds`: array of numeric tile IDs in mapget’s tiling scheme. This is an **unstaged** request shape: the service does not expand it into one backend fetch per advertised stage and returns one tile response per requested tile with no explicit stage affinity.
+    - or `tileIdsByNextStage`: array of arrays where bucket `i` lists tiles whose next missing stage is `i`. This is the **staged** request shape: the service expands each tile to stage `i` and all higher stages advertised by the layer.
   - `stringPoolOffsets` (optional): dictionary from datasource node ID to last known string ID. Used by advanced clients to avoid receiving the same field names repeatedly in the binary stream.
-  - `clientId` (optional): arbitrary string identifying this client connection for abort handling.
 - **Response:**
   - `application/jsonl` if `Accept: application/jsonl` is sent.
   - `application/binary` if `Accept: application/binary` is sent, using the tile stream protocol.
 
 Tiles are streamed as they become available. In JSONL mode, each line is the JSON representation of one tile layer. In binary mode, the response is a sequence of versioned messages that can be decoded using the tile stream protocol from `mapget-model.md`.
 
+For staged feature-layer clients, `tileIdsByNextStage` must be used even when only bucket `0` is non-empty. Collapsing such a request to plain `tileIds` changes its semantics to an unstaged request.
+
 If `Accept-Encoding: gzip` is set, the server compresses responses where possible, which is especially useful for JSONL streams.
 
+To cancel an in-flight HTTP stream, close the HTTP connection.
+
+## `/tiles` – interactive control channel (WebSocket)
+
+`GET /tiles` supports WebSocket upgrades. This endpoint is the control channel for interactive clients. It carries request updates and lightweight status/control frames; binary tile data is pulled separately via `/tiles/next`.
+
+- **Connect:** `ws://<host>:<port>/tiles`
+- **Client → Server:** send one *text* message containing the same JSON body as for `POST /tiles` (`requests`, optional `stringPoolOffsets`).
+  - `stringPoolOffsets` is optional; the server remembers the latest offsets per WebSocket connection. Clients may re-send it to reset/resync offsets.
+- **Server → Client:** sends *binary* WebSocket messages carrying VTLV control frames.
+  - `RequestContext` frames contain a UTF-8 JSON payload with `requestId` and `clientId`. The `clientId` is then used for `/tiles/next`.
+  - `Status` frames contain UTF-8 JSON describing per-request `RequestStatus` transitions and a human-readable message. The final status frame has `"allDone": true`.
+  - `LoadStateChange` exists in the protocol but is currently not emitted by the HTTP service.
+
+To cancel, either send a new request message on the same connection (which replaces the current one) or close the WebSocket connection.
+
+## `/tiles/next` – pull binary tile frames
+
+`GET /tiles/next` (also accepts `POST`) returns the next available binary tile frame batch for an active interactive `/tiles` session.
+
+- **Method:** `GET` or `POST`
+- **Query parameters:**
+  - `clientId` (required): numeric client id received via the websocket `RequestContext` frame.
+  - `waitMs` (optional): long-poll timeout in milliseconds. Defaults to 25000 and is clamped to 30000.
+  - `maxBytes` (optional): batch size budget. If greater than zero, the response may concatenate multiple VTLV frames up to that byte budget (capped at 5 MiB).
+  - `compress` (optional): set to `1` to enable gzip compression when the client also sends `Accept-Encoding: gzip`.
+- **Response:**
+  - `200 application/octet-stream` with one or more concatenated `TileLayerStream` VTLV frames.
+  - `204 No Content` if the long-poll timed out before any frame became available.
+  - `410 Gone` if the interactive session no longer exists.
+
 ### Why JSONL instead of JSON?
 
 JSON Lines is better suited to streaming large responses than a single JSON array. Clients can start processing the first tiles immediately, do not need to buffer the complete response in memory, and can naturally consume the stream with incremental parsers.
@@ -87,16 +120,6 @@ Each line in the JSONL response is a GeoJSON-like FeatureCollection with additio
 
 The `error` object is only present if an error occurred while filling the tile. When present, the `features` array may be empty or contain partial data.
 
-## `/abort` – cancel tile streaming
-
-`POST /abort` cancels a running `/tiles` request that was started with a matching `clientId`. It is useful when the viewport changes and the previous stream should be abandoned.
-
-- **Method:** `POST`
-- **Request body (JSON):** `{ "clientId": "<same-id-used-for-tiles>" }`
-- **Response:** `text/plain` confirmation; a 400 status code if `clientId` is missing.
-
-Internally the service marks the matching tile requests as aborted and stops scheduling further work for them.
-
 ### Curl Call Example
 
 For example, the following curl call could be used to stream GeoJSON feature objects
@@ -179,7 +202,24 @@ Keep in mind, that you can also run a `mapget` service without any RPCs in your
 - **Request body:** none
 - **Response:** `text/html`
 
-The page shows the number of active datasources and worker threads, the size of the active request queue and cache statistics such as hit/miss counters. This endpoint is primarily used during development and debugging.
+The page shows the number of active datasources and worker threads, cache statistics, websocket/pull metrics, and optional tile-size-distribution data. It refreshes by polling `/status-data`. This endpoint is primarily used during development and debugging.
+
+## `/status-data` – machine-readable diagnostics
+
+`GET /status-data` returns the JSON payload that powers `/status`.
+
+- **Method:** `GET`
+- **Query parameters:**
+  - `includeTileSizeDistribution` (optional, default `false`): include the heavy cached-tile size histogram / distribution calculations.
+  - `includeCachedFeatureTreeBytes` (optional, default `true`): include cached feature-tree byte breakdowns.
+- **Response:** `application/json`
+
+The response contains:
+
+- `timestampMs`
+- `service`: service statistics, datasource info, cache occupancy, and optional tile-size-distribution data
+- `cache`: cache hit/miss counters and cache sizes
+- `tilesWebsocket`: control-channel metrics such as active sessions, pending queued frames for `/tiles/next`, blocked pull requests, and total forwarded bytes / frames
 
 ## `/locate` – resolve external feature IDs
 

docs/mapget-cache.md

@@ -33,10 +33,9 @@ Advanced clients can take advantage of this by setting the `stringPoolOffsets` f
 
 ## Inspecting cache statistics
 
-The easiest way to see how the cache behaves is to call `GET /status` on the running server. The HTML status page contains:
+The easiest way to see how the cache behaves is to call `GET /status` on the running server or query `GET /status-data` directly. The HTML status page contains:
 
 - Global service information such as the number of active datasources and worker threads.
 - Cache statistics, including `cache-hits`, `cache-misses` and the number of loaded string pools.
 
 When the in‑memory cache is used, additional fields show the current number of cached tiles and the size of the FIFO queue. These values provide a quick indication of whether the chosen cache size is appropriate for the workload.
-

docs/mapget-config.md

@@ -194,7 +194,7 @@ The generator will produce deterministic but varied features for any requested t
 
 <!-- --8<-- [start:geojson] -->
 
-`GeoJsonFolder` serves tiles from a directory containing GeoJSON files.
+`GeoJsonFolder` serves tiles from a directory containing GeoJSON files. It accepts the usual GeoJSON geometry families, including `Point`, `MultiPoint`, `LineString`, `MultiLineString`, `Polygon`, `MultiPolygon`, and `GeometryCollection`.
 
 Required fields:
 
@@ -203,14 +203,16 @@ Required fields:
 
 Optional fields:
 
-- `withAttrLayers`: boolean flag. If `true`, nested objects in the GeoJSON `properties` are interpreted as attribute layers; if `false`, only scalar top‑level properties are emitted.
+- `mapId`: optional map ID override. If omitted, mapget derives a display name from the input directory.
+- `withAttrLayers` (default: `true`): boolean flag. If `true`, nested objects in the GeoJSON `properties` are converted to mapget attribute layers; if `false`, only scalar top‑level properties are emitted and nested objects are silently dropped.
 
 Example:
 
 ```yaml
 sources:
   - type: GeoJsonFolder
     folder: /data/tiles
+    mapId: Road Test Data
     withAttrLayers: true
 ```