diff --git a/src/core/Application.cpp b/src/core/Application.cpp index 478346a8..60c74d5e 100644 --- a/src/core/Application.cpp +++ b/src/core/Application.cpp @@ -83,6 +83,19 @@ #include #include #include +#include +#include +#include +#include "../utils/ShaderBundle.h" // #54 shader bundle transport + +// #54 — FFmpeg avio for the binary shader-bundle GET (http/https), same +// transport the /meta poller already uses. +extern "C" { +#include +#include +#include +#include +} // swscale removed - resize is now done in encoding (HTTPTSStreamer) @@ -92,6 +105,9 @@ Application::Application() Application::~Application() { + // #54 — make sure the shader-bundle worker is done before members it + // touches go away. + if (m_shaderBundleThread.joinable()) m_shaderBundleThread.join(); shutdown(); } @@ -2807,6 +2823,15 @@ bool Application::initUI() authToken = m_ui->getRemoteAuthToken(); m_ui->setRemoteAuthToken(""); } + // #54 — keep the token for the shader-bundle fetch (same auth as + // the stream / /meta), and reset the per-connection fetch state so + // a new stream re-evaluates its shader from scratch. + { + std::lock_guard lock(m_shaderBundleMutex); + m_remoteAuthToken = authToken; + } + m_shaderBundleLastTriedHash.clear(); + m_appliedRemotePresetHash.clear(); // Recreate the capture as Remote (the existing instance might be // V4L2/DS if the user just flipped source type). @@ -6224,6 +6249,184 @@ std::string Application::resolveShaderPath(const std::string &shaderPath) const return fullPath.string(); } +namespace +{ + // #54 — binary HTTP(S) GET via FFmpeg avio (mirrors RemoteMetaSync's + // text GET). Returns the raw body, capped so a misbehaving host can't + // make us allocate unbounded. + std::string httpGetBinary(const std::string &url, const std::string &authToken, + size_t maxBytes, int recvTimeoutMs = 8000) + { + avformat_network_init(); + AVDictionary *opts = nullptr; + av_dict_set_int(&opts, "rw_timeout", static_cast(recvTimeoutMs) * 1000, 0); + av_dict_set(&opts, "user_agent", "RetroCapture/0.8", 0); + std::string headers = "Accept: application/octet-stream\r\n"; + if (!authToken.empty()) + { + headers += "Authorization: Bearer " + authToken + "\r\n"; + } + av_dict_set(&opts, "headers", headers.c_str(), 0); + + AVIOContext *io = nullptr; + int rc = avio_open2(&io, url.c_str(), AVIO_FLAG_READ, nullptr, &opts); + av_dict_free(&opts); + if (rc < 0) + { + char errbuf[256] = {}; + av_strerror(rc, errbuf, sizeof(errbuf)); + LOG_WARN(std::string("ShaderBundle: avio_open2 failed for ") + url + ": " + errbuf); + return {}; + } + std::string body; + unsigned char buf[8192]; + for (;;) + { + int n = avio_read(io, buf, sizeof(buf)); + if (n <= 0) break; + body.append(reinterpret_cast(buf), static_cast(n)); + if (body.size() > maxBytes) { body.clear(); break; } // over cap → reject + } + avio_closep(&io); + return body; + } + + // Hash like "fnv1a64:abcd…" → a filesystem-safe directory name. + std::string hashToDirName(const std::string &hash) + { + std::string s = hash; + for (char &c : s) + if (c == ':' || c == '/' || c == '\\') c = '_'; + return s; + } +} + +std::string Application::shaderBundleCacheRoot(const std::string &presetHash) const +{ + return Paths::getCacheDir() + "/shader-cache/" + hashToDirName(presetHash); +} + +std::string Application::cachedShaderBundleGlslp(const std::string &presetHash) const +{ + if (presetHash.empty()) return {}; + const std::string root = shaderBundleCacheRoot(presetHash); + // The fetch writes a tiny pointer file with the bundle's .glslp rel-path + // so we don't have to scan/iterate the cache dir (portable across the + // FilesystemCompat shim). + std::ifstream idx(root + "/.bundle-glslp", std::ios::binary); + if (!idx.is_open()) return {}; + std::string rel; + std::getline(idx, rel); + while (!rel.empty() && (rel.back() == '\n' || rel.back() == '\r')) rel.pop_back(); + if (rel.empty()) return {}; + const std::string full = root + "/" + rel; + if (!fs::exists(full)) return {}; + return full; +} + +void Application::requestShaderBundleFetch(const std::string &presetHash, + const std::string &presetName) +{ + if (presetHash.empty()) return; + if (m_shaderBundleFetching.load()) return; // one fetch at a time + if (presetHash == m_shaderBundleLastTriedHash) return; // don't hammer a failing hash + + const std::string baseUrl = m_devicePath; // remote base URL (Remote source) + if (baseUrl.empty()) return; + std::string token; + { + std::lock_guard lock(m_shaderBundleMutex); + token = m_remoteAuthToken; + } + + m_shaderBundleLastTriedHash = presetHash; + m_shaderBundleFetching.store(true); + LOG_INFO("ShaderBundle: fetching host shader bundle (hash " + presetHash + ")"); + + // Reap the previous (already-finished) worker before starting a new one. + if (m_shaderBundleThread.joinable()) m_shaderBundleThread.join(); + m_shaderBundleThread = std::thread( + [this, presetHash, presetName, baseUrl, token]() + { + std::string glslpPath; + const bool ok = fetchAndCacheShaderBundle(baseUrl, token, presetHash, glslpPath); + if (ok) + { + std::lock_guard lock(m_shaderBundleMutex); + m_pendingShaderBundleGlslp = glslpPath; + m_pendingShaderBundleHash = presetHash; + m_pendingShaderBundlePreset = presetName; + m_hasPendingShaderBundle.store(true); + } + else + { + LOG_WARN("ShaderBundle: fetch/cache failed (hash " + presetHash + ")"); + } + m_shaderBundleFetching.store(false); + }); +} + +bool Application::fetchAndCacheShaderBundle(const std::string &baseUrl, + const std::string &authToken, + const std::string &presetHash, + std::string &glslpPathOut) +{ + const std::string url = baseUrl + "/api/v1/shader/bundle"; + const std::string body = httpGetBinary(url, authToken, 16ull * 1024 * 1024); + if (body.empty()) return false; + + std::vector entries; + if (!shaderbundle::unpack(body, entries) || entries.empty()) + { + LOG_WARN("ShaderBundle: failed to unpack bundle from host"); + return false; + } + + const std::string root = shaderBundleCacheRoot(presetHash); + std::string glslpRel; + for (const auto &e : entries) + { + const std::string dest = root + "/" + e.relPath; + // Parent dir via plain string ops (avoids relying on fs::path::string() + // which the FilesystemCompat shim doesn't expose). + std::string parent = dest; + size_t slash = parent.find_last_of("/\\"); + if (slash != std::string::npos) parent.resize(slash); else parent.clear(); + // Single-arg form for FilesystemCompat (the shim has no error_code + // overload); std::filesystem throws on error, so guard both. + if (!parent.empty()) { try { fs::create_directories(parent); } catch (...) {} } + std::ofstream f(dest, std::ios::binary | std::ios::trunc); + if (!f.is_open()) + { + LOG_WARN("ShaderBundle: cannot write " + dest); + return false; + } + f.write(e.data.data(), static_cast(e.data.size())); + f.close(); + + // The .glslp is the preset entry point. + if (glslpRel.empty() && e.relPath.size() >= 6 && + e.relPath.compare(e.relPath.size() - 6, 6, ".glslp") == 0) + { + glslpRel = e.relPath; + } + } + if (glslpRel.empty()) + { + LOG_WARN("ShaderBundle: bundle has no .glslp entry"); + return false; + } + // Pointer file so a later session resolves the bundle without scanning. + { + std::ofstream idx(root + "/.bundle-glslp", std::ios::binary | std::ios::trunc); + if (idx.is_open()) idx << glslpRel << "\n"; + } + glslpPathOut = root + "/" + glslpRel; + LOG_INFO("ShaderBundle: cached " + std::to_string(entries.size()) + + " files to " + root); + return true; +} + void Application::applyPendingRemoteMeta() { if (!m_hasPendingRemoteMeta.load()) return; @@ -6309,21 +6512,61 @@ void Application::applyPendingRemoteMeta() bool reloaded = false; if (!preset.empty() && presetHash != m_appliedRemotePresetHash) { + // 1) Resolve the preset by name in the local shader library. + bool loaded = false; const std::string fullPath = resolveShaderPath(preset); if (!fullPath.empty()) { LOG_INFO("RemoteMetaSync: applying host preset '" + preset + "' (hash " + presetHash + ")"); - if (m_shaderEngine->loadPreset(fullPath)) - { - m_ui->setCurrentShader(preset); - m_appliedRemotePresetHash = presetHash; - reloaded = true; - } - else + loaded = m_shaderEngine->loadPreset(fullPath); + } + // 2) #54 — not present locally (or failed to load): try a bundle we + // already fetched for this hash, otherwise fetch it from the host. + if (!loaded) + { + const std::string cached = cachedShaderBundleGlslp(presetHash); + if (!cached.empty() && m_shaderEngine->loadPreset(cached)) { - LOG_WARN("RemoteMetaSync: failed to load preset locally — bundle fetch is a Phase 4b TODO"); + LOG_INFO("RemoteMetaSync: applied cached shader bundle (hash " + presetHash + ")"); + loaded = true; } } + if (loaded) + { + m_ui->setCurrentShader(preset); + m_appliedRemotePresetHash = presetHash; + reloaded = true; + } + else + { + requestShaderBundleFetch(presetHash, preset); + } + } + + // #54 — a background bundle fetch finished: load it on this (GL) thread, + // where ShaderEngine/GL access is safe. + if (m_hasPendingShaderBundle.load()) + { + std::string glslp, hash, name; + { + std::lock_guard lock(m_shaderBundleMutex); + glslp = std::move(m_pendingShaderBundleGlslp); + hash = std::move(m_pendingShaderBundleHash); + name = std::move(m_pendingShaderBundlePreset); + m_pendingShaderBundleGlslp.clear(); + m_hasPendingShaderBundle.store(false); + } + if (!glslp.empty() && m_shaderEngine && m_shaderEngine->loadPreset(glslp)) + { + if (m_ui && !name.empty()) m_ui->setCurrentShader(name); + m_appliedRemotePresetHash = hash; + reloaded = true; + LOG_INFO("RemoteMetaSync: applied fetched shader bundle '" + name + "' (hash " + hash + ")"); + } + else + { + LOG_WARN("RemoteMetaSync: fetched shader bundle failed to load (hash " + hash + ")"); + } } // Master pipeline toggle: mirror the host's "Apply shader pipeline". diff --git a/src/core/Application.h b/src/core/Application.h index dc50a1dc..1e745b7b 100644 --- a/src/core/Application.h +++ b/src/core/Application.h @@ -153,6 +153,16 @@ class Application // Shader path resolution (centralized) std::string resolveShaderPath(const std::string& shaderPath) const; + // #54 — shader bundle fetch for remote (directory) streams. + std::string shaderBundleCacheRoot(const std::string &presetHash) const; + std::string cachedShaderBundleGlslp(const std::string &presetHash) const; + void requestShaderBundleFetch(const std::string &presetHash, + const std::string &presetName); + bool fetchAndCacheShaderBundle(const std::string &baseUrl, + const std::string &authToken, + const std::string &presetHash, + std::string &glslpPathOut); + // Phase 4 of #47: drains pending remote /meta snapshot onto the GL // thread. Called once per main-loop iteration; cheap no-op when // m_hasPendingRemoteMeta is false. @@ -272,6 +282,21 @@ class Application std::string m_pendingRemotePreset; std::string m_pendingRemotePresetHash; std::string m_appliedRemotePresetHash; + + // #54 — shader-bundle fetch. When the host announces a shader the client + // doesn't have locally, fetch the bundle (.glslp + .glsl + LUTs) off the + // GL thread, cache it under getCacheDir()/shader-cache//, and apply + // it on a later tick. m_remoteAuthToken keeps the connect's bearer token + // around for the fetch (the UI copy is consumed once at connect). + std::string m_remoteAuthToken; + std::thread m_shaderBundleThread; + std::mutex m_shaderBundleMutex; + std::atomic m_shaderBundleFetching{false}; + std::string m_shaderBundleLastTriedHash; + std::atomic m_hasPendingShaderBundle{false}; + std::string m_pendingShaderBundleGlslp; + std::string m_pendingShaderBundleHash; + std::string m_pendingShaderBundlePreset; bool m_pendingRemotePipelineEnabled = true; std::vector> m_pendingRemoteParams; // Host's source resolution from /meta. The capture rescales /raw to diff --git a/src/streaming/APIController.cpp b/src/streaming/APIController.cpp index 285aca72..04078027 100644 --- a/src/streaming/APIController.cpp +++ b/src/streaming/APIController.cpp @@ -2,6 +2,7 @@ #include "../core/Application.h" #include "../ui/UIManager.h" #include "../shader/ShaderEngine.h" +#include "../shader/ShaderPreset.h" #include "HTTPServer.h" #include "../utils/HttpAuth.h" #include "../utils/Logger.h" @@ -380,6 +381,29 @@ ssize_t APIController::sendData(int clientFd, const void *data, size_t size) con return m_httpServer->sendData(clientFd, data, size); } +bool APIController::sendAll(int clientFd, const char *data, size_t size) const +{ + size_t off = 0; + int idleSpins = 0; + while (off < size) + { + ssize_t s = sendData(clientFd, data + off, size - off); + if (s < 0) return false; // peer gone / fatal socket error + if (s == 0) + { + // EAGAIN/EWOULDBLOCK — socket send buffer is full. Back off and + // retry; bail after a long stall (~30 s) so a dead client can't + // wedge the handler forever. + if (++idleSpins > 30000) return false; + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + continue; + } + idleSpins = 0; + off += static_cast(s); + } + return true; +} + bool APIController::handleGET(int clientFd, const std::string &path, const std::string &request) { if (path == "/meta") @@ -406,6 +430,10 @@ bool APIController::handleGET(int clientFd, const std::string &path, const std:: { return handleGETShaderParameters(clientFd); } + else if (path == "/api/v1/shader/bundle") + { + return handleGETShaderBundle(clientFd, request); + } else if (path == "/api/v1/capture/resolution") { return handleGETCaptureResolution(clientFd); @@ -1315,16 +1343,236 @@ bool APIController::handleGETMetaSSE(int clientFd) return true; } +namespace +{ + // Read a whole file into a string. Empty string on failure. + std::string readFileBytes(const std::string &path) + { + std::ifstream f(path, std::ios::binary); + if (!f.is_open()) return {}; + std::ostringstream ss; + ss << f.rdbuf(); + return ss.str(); + } + + // Split a path into its components on '/' or '\\' (drops empties). Kept + // string-based so it works with both std::filesystem and the minimal + // FilesystemCompat shim used on the MinGW build (which lacks relative()/ + // weakly_canonical()/path iterators). + std::vector pathComponents(const std::string &p) + { + std::vector comps; + std::string cur; + for (char c : p) + { + if (c == '/' || c == '\\') + { + if (!cur.empty()) { comps.push_back(cur); cur.clear(); } + } + else { cur += c; } + } + if (!cur.empty()) comps.push_back(cur); + return comps; + } +} + +bool APIController::collectShaderBundle(const std::string &presetPath, + std::vector &out, + std::string &rootRelGlslp, + uint64_t &totalBytes) const +{ + out.clear(); + rootRelGlslp.clear(); + totalBytes = 0; + if (presetPath.empty() || !fs::exists(presetPath)) + { + LOG_WARN("collectShaderBundle: preset path missing/unreadable: " + presetPath); + return false; + } + + // Parse the preset to enumerate every file it references. + ShaderPreset preset; + if (!preset.load(presetPath)) + { + LOG_WARN("collectShaderBundle: ShaderPreset::load failed for " + presetPath); + return false; + } + + // Gather the files (the .glslp + pass shaders + LUT textures), deduped. + // ShaderPreset has already resolved these to usable paths. + std::vector paths; + auto addPath = [&](const std::string &p) { + if (p.empty()) return; + if (std::find(paths.begin(), paths.end(), p) == paths.end()) + paths.push_back(p); + }; + addPath(presetPath); + const size_t glslpIdx = 0; // presetPath is paths[0] + for (const auto &pass : preset.getPasses()) addPath(pass.shaderPath); + for (const auto &kv : preset.getTextures()) addPath(kv.second.path); + + // Common root = longest common prefix of the path components, so the + // relative paths never need '..' and the client can extract them under + // one cache dir while preserving the preset's relative references. + std::vector> comps; + comps.reserve(paths.size()); + size_t minLen = SIZE_MAX; + for (const auto &p : paths) + { + comps.push_back(pathComponents(p)); + minLen = std::min(minLen, comps.back().size()); + } + if (minLen == 0) return false; + size_t commonLen = 0; + for (size_t i = 0; i < minLen; ++i) + { + const std::string &c = comps[0][i]; + bool allMatch = true; + for (const auto &cv : comps) { if (cv[i] != c) { allMatch = false; break; } } + if (!allMatch) break; + ++commonLen; + } + // Keep at least the filename in every relative path. + if (commonLen >= minLen) commonLen = minLen - 1; + + for (size_t idx = 0; idx < paths.size(); ++idx) + { + std::string relStr; + for (size_t i = commonLen; i < comps[idx].size(); ++i) + { + if (!relStr.empty()) relStr += '/'; + relStr += comps[idx][i]; + } + if (!shaderbundle::relPathIsSafe(relStr)) + { + LOG_WARN("collectShaderBundle: unsafe rel path '" + relStr + + "' from '" + paths[idx] + "'"); + return false; + } + + std::string bytes = readFileBytes(paths[idx]); + if (bytes.empty() && !fs::exists(paths[idx])) + { + LOG_WARN("collectShaderBundle: referenced file missing: " + paths[idx]); + return false; + } + totalBytes += bytes.size(); + if (totalBytes > kMaxShaderBundleBytes) + { + LOG_WARN("collectShaderBundle: bundle exceeds cap (" + + std::to_string(totalBytes) + " > " + + std::to_string(kMaxShaderBundleBytes) + " bytes) at " + paths[idx]); + return false; + } + + shaderbundle::Entry e; + e.relPath = relStr; + e.data = std::move(bytes); + if (idx == glslpIdx) rootRelGlslp = relStr; + out.push_back(std::move(e)); + } + if (rootRelGlslp.empty() && !out.empty()) rootRelGlslp = out.front().relPath; + LOG_INFO("collectShaderBundle: " + std::to_string(out.size()) + " files, " + + std::to_string(totalBytes) + " bytes, glslp='" + rootRelGlslp + "'"); + return !out.empty(); +} + std::string APIController::computePresetHash(const std::string &presetPath) const { - std::ifstream file(presetPath, std::ios::binary); - if (!file.is_open()) + if (presetPath.empty()) return ""; + + // Cache the bundle hash so the ~1 Hz /meta poll doesn't re-read every + // shader file each time. Keyed by preset path; switching presets + // recomputes. (An in-place edit of the same .glslp won't refresh the + // hash until a different preset is selected — acceptable: shaders are + // static assets, rarely edited mid-stream.) { - return ""; + std::lock_guard lock(m_bundleHashMu); + if (presetPath == m_bundleHashPath && !m_bundleHashValue.empty()) + { + return m_bundleHashValue; + } + } + + std::vector entries; + std::string rootRelGlslp; + uint64_t totalBytes = 0; + std::string hash; + if (collectShaderBundle(presetPath, entries, rootRelGlslp, totalBytes)) + { + hash = shaderbundle::hashEntries(entries); + } + else + { + // Fall back to hashing just the .glslp (e.g. bundle over the size + // cap) so /meta still announces a stable identity. + hash = shaderbundle::fnv1a64Hex(readFileBytes(presetPath)); + } + + { + std::lock_guard lock(m_bundleHashMu); + m_bundleHashPath = presetPath; + m_bundleHashValue = hash; + } + return hash; +} + +bool APIController::handleGETShaderBundle(int clientFd, const std::string &request) +{ + (void)request; + if (!m_application) + { + sendErrorResponse(clientFd, 500, "Application not available"); + return true; + } + + ShaderEngine *shaderEngine = m_application->getShaderEngine(); + const std::string presetPath = shaderEngine ? shaderEngine->getPresetPath() : ""; + if (presetPath.empty()) + { + sendErrorResponse(clientFd, 404, "No active shader preset"); + return true; } - std::ostringstream buffer; - buffer << file.rdbuf(); - return fnv1a64Hex(buffer.str()); + + std::vector entries; + std::string rootRelGlslp; + uint64_t totalBytes = 0; + if (!collectShaderBundle(presetPath, entries, rootRelGlslp, totalBytes)) + { + // Either unreadable or over the size cap — tell the client so it can + // show "shader not available" instead of hanging. + sendErrorResponse(clientFd, 413, "Shader bundle unavailable or too large"); + return true; + } + + const std::string blob = shaderbundle::pack(entries); + const std::string hash = shaderbundle::hashEntries(entries); + + std::ostringstream response; + response << "HTTP/1.1 200 OK\r\n"; + response << "Content-Type: application/octet-stream\r\n"; + response << "Content-Length: " << blob.size() << "\r\n"; + // The bundle's glslp entry path + content hash, so the client knows what + // to load and how to key its cache without re-deriving them. + response << "X-Shader-Bundle-Glslp: " << rootRelGlslp << "\r\n"; + response << "X-Shader-Bundle-Hash: " << hash << "\r\n"; + response << "Connection: close\r\n\r\n"; + + const std::string headerStr = response.str(); + if (!sendAll(clientFd, headerStr.c_str(), headerStr.size())) return true; + + // Stream the blob in chunks. sendAll handles partial/EAGAIN sends — a + // single send() would silently drop the tail of a chunk once the socket + // buffer fills, corrupting the bundle (same bug fixed for recordings). + const size_t chunk = 64 * 1024; + for (size_t off = 0; off < blob.size(); off += chunk) + { + const size_t n = std::min(chunk, blob.size() - off); + if (!sendAll(clientFd, blob.data() + off, n)) return true; + } + LOG_INFO("APIController: served shader bundle (" + std::to_string(entries.size()) + + " files, " + std::to_string(blob.size()) + " bytes, hash " + hash + ")"); + return true; } bool APIController::handleGETPlatform(int clientFd) diff --git a/src/streaming/APIController.h b/src/streaming/APIController.h index 53ba8dab..912cbf87 100644 --- a/src/streaming/APIController.h +++ b/src/streaming/APIController.h @@ -4,6 +4,9 @@ #include #include #include +#include + +#include "../utils/ShaderBundle.h" class UIManager; class Application; @@ -105,6 +108,14 @@ class APIController */ ssize_t sendData(int clientFd, const void *data, size_t size) const; + /** + * Reliably send the entire buffer. sendData() does a single non-blocking + * send() that can return a partial count or 0 (EAGAIN, socket buffer + * full); this loops until everything is sent so large responses aren't + * silently truncated. Returns false on a fatal socket error. + */ + bool sendAll(int clientFd, const char *data, size_t size) const; + /** * Compute a content hash of a preset file for the /meta endpoint. * Returns an opaque string (e.g. "fnv1a64:abcd...") used by the remote @@ -113,6 +124,18 @@ class APIController */ std::string computePresetHash(const std::string &presetPath) const; + // #54 — collect a preset's bundle (the .glslp + every .glsl/LUT it + // references), with paths relative to their common root so the layout + // survives client extraction. Returns false if the preset can't be read + // or the bundle exceeds kMaxShaderBundleBytes. Used for both the bundle + // hash (in /meta) and the /api/v1/shader/bundle endpoint. + bool collectShaderBundle(const std::string &presetPath, + std::vector &out, + std::string &rootRelGlslp, + uint64_t &totalBytes) const; + bool handleGETShaderBundle(int clientFd, const std::string &request); + static constexpr uint64_t kMaxShaderBundleBytes = 8ull * 1024 * 1024; // 8 MB cap + // Endpoints GET (leitura) bool handleGET(int clientFd, const std::string &path, const std::string &request); bool handleGETSource(int clientFd); @@ -224,4 +247,12 @@ class APIController // #49 Phase 3 — sha256(password) hex; empty == no auth. mutable std::mutex m_passwordMu; std::string m_streamPasswordHash; + + // #54 — cache of the active preset's bundle hash so the ~1 Hz /meta + // poll doesn't re-read every shader file each time. Invalidated when the + // preset path or the .glslp's mtime changes. + mutable std::mutex m_bundleHashMu; + mutable std::string m_bundleHashPath; // presetPath the cache is for + mutable int64_t m_bundleHashMtime = 0; + mutable std::string m_bundleHashValue; // cached bundle hash }; diff --git a/src/utils/ShaderBundle.h b/src/utils/ShaderBundle.h new file mode 100644 index 00000000..e803559b --- /dev/null +++ b/src/utils/ShaderBundle.h @@ -0,0 +1,167 @@ +#pragma once + +// #54 — shader bundle transport for directory streams. +// +// A "bundle" is a shader preset (.glslp) plus every file it references +// (.glsl passes + LUT textures), packed into one blob so a remote client +// that doesn't have the host's shader can fetch and reproduce it. This is a +// transport only — the shader format is unchanged; files are carried +// verbatim with their paths relative to a common root so the directory +// layout (and thus the preset's relative references) survives extraction. +// +// Format ("RCSB1"): magic, then for each entry +// uint32 pathLen (LE) | path bytes | uint64 dataLen (LE) | data bytes +// Header-only and dependency-free so both the host (APIController) and the +// client (Application) share one implementation. + +#include +#include +#include +#include +#include + +namespace shaderbundle +{ + +struct Entry +{ + std::string relPath; // path relative to the bundle root (POSIX '/') + std::string data; // raw file bytes +}; + +// Magic prefix "RCSB1". A function (not an inline constexpr array, which the +// project's older MinGW toolchain rejects) so the header stays C++11-clean. +inline const char *magic() { return "RCSB1"; } +inline std::size_t magicLen() { return 5; } + +// FNV-1a 64-bit, hex. Not cryptographic — only a content key for the +// shader cache (matches the rest of the codebase's hashing choice). +inline std::string fnv1a64Hex(const std::string &content) +{ + const uint64_t FNV_OFFSET = 0xcbf29ce484222325ULL; + const uint64_t FNV_PRIME = 0x100000001b3ULL; + uint64_t h = FNV_OFFSET; + for (unsigned char c : content) + { + h ^= c; + h *= FNV_PRIME; + } + static const char *hex = "0123456789abcdef"; + std::string out = "fnv1a64:"; + for (int shift = 60; shift >= 0; shift -= 4) + out += hex[(h >> shift) & 0xF]; + return out; +} + +// Content hash over the whole bundle: entries sorted by relPath, each +// contributing relPath + '\0' + data. Deterministic regardless of the +// order files were collected, so the same preset always hashes the same. +inline std::string hashEntries(std::vector entries) +{ + std::sort(entries.begin(), entries.end(), + [](const Entry &a, const Entry &b) { return a.relPath < b.relPath; }); + std::string acc; + for (const auto &e : entries) + { + acc += e.relPath; + acc.push_back('\0'); + acc += e.data; + } + return fnv1a64Hex(acc); +} + +namespace detail +{ + inline void putU32(std::string &s, uint32_t v) + { + char b[4] = {char(v & 0xFF), char((v >> 8) & 0xFF), + char((v >> 16) & 0xFF), char((v >> 24) & 0xFF)}; + s.append(b, 4); + } + inline void putU64(std::string &s, uint64_t v) + { + char b[8]; + for (int i = 0; i < 8; ++i) b[i] = char((v >> (8 * i)) & 0xFF); + s.append(b, 8); + } + inline bool getU32(const std::string &s, size_t &pos, uint32_t &out) + { + if (pos + 4 > s.size()) return false; + out = uint32_t(uint8_t(s[pos])) | (uint32_t(uint8_t(s[pos + 1])) << 8) | + (uint32_t(uint8_t(s[pos + 2])) << 16) | (uint32_t(uint8_t(s[pos + 3])) << 24); + pos += 4; + return true; + } + inline bool getU64(const std::string &s, size_t &pos, uint64_t &out) + { + if (pos + 8 > s.size()) return false; + out = 0; + for (int i = 0; i < 8; ++i) out |= uint64_t(uint8_t(s[pos + i])) << (8 * i); + pos += 8; + return true; + } +} // namespace detail + +inline std::string pack(const std::vector &entries) +{ + std::string out(magic(), magicLen()); + for (const auto &e : entries) + { + detail::putU32(out, static_cast(e.relPath.size())); + out += e.relPath; + detail::putU64(out, static_cast(e.data.size())); + out += e.data; + } + return out; +} + +// Reject paths that could escape the extraction root (absolute, drive +// letters, or any '..' component). The bundle is auth-gated, but a client +// must never write outside its cache dir regardless. +inline bool relPathIsSafe(const std::string &p) +{ + if (p.empty() || p.size() > 1024) return false; + if (p.front() == '/' || p.front() == '\\') return false; + if (p.size() >= 2 && p[1] == ':') return false; // C:\... + size_t start = 0; + for (size_t i = 0; i <= p.size(); ++i) + { + if (i == p.size() || p[i] == '/' || p[i] == '\\') + { + std::string comp = p.substr(start, i - start); + if (comp == "..") return false; + start = i + 1; + } + } + return true; +} + +inline bool unpack(const std::string &blob, std::vector &out) +{ + out.clear(); + if (blob.size() < magicLen() || + std::memcmp(blob.data(), magic(), magicLen()) != 0) + { + return false; + } + size_t pos = magicLen(); + while (pos < blob.size()) + { + uint32_t pathLen = 0; + if (!detail::getU32(blob, pos, pathLen)) return false; + if (pos + pathLen > blob.size()) return false; + Entry e; + e.relPath = blob.substr(pos, pathLen); + pos += pathLen; + uint64_t dataLen = 0; + if (!detail::getU64(blob, pos, dataLen)) return false; + if (pos + dataLen > blob.size()) return false; + e.data = blob.substr(pos, static_cast(dataLen)); + pos += static_cast(dataLen); + if (!relPathIsSafe(e.relPath)) return false; + out.push_back(std::move(e)); + } + return true; +} + +} // namespace shaderbundle