From 986f1d69f75e72f2f0f6e9e452ceedbcb027cb4f Mon Sep 17 00:00:00 2001 From: Aster Seker Date: Sat, 19 Sep 2026 20:59:02 +0300 Subject: [PATCH] feat(readers): support transparent compressed file reads Decompress persisted gzip and zstd artifacts through the existing FileLogger and UniqueFileLogger read APIs when the matching feature is enabled. Keep disabled-feature and malformed-input paths fail-closed, cover single and batch reads, and document the contract in the README, reference docs, roadmap, changelog, and ADR 0007. --- CHANGELOG.md | 5 +- README-RU.md | 9 ++-- README.md | 12 ++--- .../0007-transparent-compressed-file-reads.md | 47 +++++++++++++++++++ docs/adr/README.md | 1 + docs/future-plans.md | 7 +-- docs/reference.dox | 5 +- .../logit/detail/CompressionUtils.hpp | 26 +++++++++- include/logit_cpp/logit/loggers.hpp | 2 +- .../logit_cpp/logit/loggers/FileLogger.hpp | 21 ++++++--- .../logit/loggers/UniqueFileLogger.hpp | 15 ++++-- tests/file_logger_gzip_compression_test.cpp | 13 ++++- tests/file_logger_zstd_compression_test.cpp | 13 ++++- tests/unique_file_logger_file_api_test.cpp | 21 ++++++++- 14 files changed, 162 insertions(+), 35 deletions(-) create mode 100644 docs/adr/0007-transparent-compressed-file-reads.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 3ecdc44d..ab620386 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,10 @@ All notable changes to this project will be documented in this file. ## [Unreleased] -Changes since v1.0.2 will be documented here. +### Added + +- Added transparent `.gz` and `.zst` reads for persisted file APIs when the + corresponding compression feature is enabled. ## [v1.0.2] - 2026-09-19 diff --git a/README-RU.md b/README-RU.md index 3b883ba2..0be0c504 100644 --- a/README-RU.md +++ b/README-RU.md @@ -223,10 +223,11 @@ int main() { Файловые backend-ы также поддерживают доступ к уже сохранённым логам через `LOGIT_LIST_LOG_FILES(index)`, `LOGIT_READ_LOG_FILE(index, path)` и `LOGIT_READ_LOG_FILES(index, paths)`. Эти helpers читают только то, что уже -успело попасть на диск, не дренируют асинхронные очереди и пока рассматривают -сжатые rotated-файлы как metadata-only записи. Используйте `MemoryLogger` для -почти real-time снимков, а файловые API — для операционного чтения логов за -сегодня или предыдущие дни. +успело попасть на диск, и не дренируют асинхронные очереди. Rotated-файлы `.gz` +и `.zst` распаковываются, когда включена соответствующая feature; иначе read +result возвращает `ok == false`. Используйте `MemoryLogger` для почти real-time +снимков, а файловые API — для операционного чтения логов за сегодня или +предыдущие дни. ### Структурированные и telemetry-бэкенды diff --git a/README.md b/README.md index 9ab942b1..8e30e5a1 100644 --- a/README.md +++ b/README.md @@ -310,10 +310,10 @@ storage should prefer the shared `LOGIT_READ_*` and callback macros above. File-based backends also expose persisted-file access through `LOGIT_LIST_LOG_FILES(index)`, `LOGIT_READ_LOG_FILE(index, path)`, and `LOGIT_READ_LOG_FILES(index, paths)`. These helpers read only what has already -reached disk, do not drain async queues, and currently treat compressed rotated -files as metadata-only entries. Use `MemoryLogger` for near-real-time snapshots -and the file APIs for operational reads of today's or previous days' persisted -logs. +reached disk and do not drain async queues. Rotated `.gz` and `.zst` files are +decompressed when the corresponding feature is enabled; otherwise their read +result has `ok == false`. Use `MemoryLogger` for near-real-time snapshots and +the file APIs for operational reads of today's or previous days' persisted logs. --- @@ -1011,8 +1011,8 @@ the rows above document the canonical public families. | `LOGIT_GET_BUFFERED_STRINGS(index)` | Return buffered formatted messages from a logger that supports snapshots. | | `LOGIT_GET_BUFFERED_ENTRIES(index)` | Return buffered structured entries from a logger that supports snapshots. | | `LOGIT_LIST_LOG_FILES(index)` | List persisted log files exposed by a file-based logger. | -| `LOGIT_READ_LOG_FILE(index, path)` | Read one persisted plain-text log file owned by a file-based logger. | -| `LOGIT_READ_LOG_FILES(index, paths)` | Read several persisted plain-text log files and preserve request order. | +| `LOGIT_READ_LOG_FILE(index, path)` | Read one persisted plain or feature-enabled compressed log file owned by a file-based logger. | +| `LOGIT_READ_LOG_FILES(index, paths)` | Read several persisted plain or feature-enabled compressed log files and preserve request order. | | `LOGIT_WAIT()` | Wait for all asynchronous loggers to finish. | | `LOGIT_SHUTDOWN()` | Shut down the logging system. | diff --git a/docs/adr/0007-transparent-compressed-file-reads.md b/docs/adr/0007-transparent-compressed-file-reads.md new file mode 100644 index 00000000..e80c6466 --- /dev/null +++ b/docs/adr/0007-transparent-compressed-file-reads.md @@ -0,0 +1,47 @@ +# ADR 0007: Transparent compressed-file reads + +- Status: Accepted +- Date: 2026-09-19 + +## Context + +`FileLogger` and `UniqueFileLogger` already expose rotated `.gz` and `.zst` +artifacts through `list_log_files()`, but their read APIs previously returned +`ok == false` for every compressed file. Callers therefore had to inspect the +suffix and duplicate LogIt++'s optional-dependency handling. + +## Decision + +`read_log_file()` and `read_log_files()` transparently decompress `.gz` and +`.zst` file contents through the shared compression helpers. Suffixes remain +case-sensitive and match the names produced by the rotation code. + +The operation succeeds only when the corresponding build feature is enabled +and the payload is valid. A disabled feature, unsupported suffix, malformed +payload, or file-read failure returns the existing `LogFileReadResult` contract +with `ok == false` and empty content. The readers do not invoke external +commands and do not drain pending asynchronous writes. + +## Consequences + +Callers use the same file-read API for plain and compressed artifacts, while +builds without zlib or zstd retain fail-closed behavior. Decompression loads the +compressed and decompressed contents into memory, matching the existing +whole-file result type; callers should avoid using this API for unbounded files. + +## Alternatives considered + +- Returning compressed bytes would make the result meaning depend on the + suffix and leave dependency handling to every caller. +- Shelling out to gzip or zstd would add platform-specific process and quoting + behavior to a synchronous read API. +- Adding separate compressed-read methods would duplicate the existing file + discovery, ownership, and batch-result contracts. + +## References + +- `include/logit_cpp/logit/loggers/FileLogger.hpp` +- `include/logit_cpp/logit/loggers/UniqueFileLogger.hpp` +- `tests/file_logger_gzip_compression_test.cpp` +- `tests/file_logger_zstd_compression_test.cpp` + diff --git a/docs/adr/README.md b/docs/adr/README.md index 9e9ac388..c58a45f8 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -29,3 +29,4 @@ explains **why the current boundary or trade-off exists**. - [0004 — TimeShield compatibility and dependency reuse](0004-timeshield-compatibility.md) - [0005 — Benchmark evidence and comparison methodology](0005-benchmark-methodology.md) - [0006 — Explicit capability for concurrent dispatch](0006-concurrent-dispatch-capability.md) +- [0007 — Transparent compressed-file reads](0007-transparent-compressed-file-reads.md) diff --git a/docs/future-plans.md b/docs/future-plans.md index da6eac31..125f40f6 100644 --- a/docs/future-plans.md +++ b/docs/future-plans.md @@ -45,9 +45,10 @@ Legend: - [ ] **Binary logging research** — choose a versioned binary record format, define compatibility/versioning rules, and prototype a reader before adding a production backend. -- [ ] **Transparent compressed-file reads** — make `read_log_file()` and - `read_log_files()` read `.gz`/`.zst` entries when the corresponding feature is - enabled, with platform-specific tests. +- [x] **Transparent compressed-file reads** — `read_log_file()` and + `read_log_files()` now decompress `.gz`/`.zst` entries when the corresponding + feature is enabled; disabled-feature and malformed-input paths remain + failures and are covered by tests. - [ ] **Configuration loading** — design a versioned JSON/properties mapping to the existing backend configuration. Treat file watching/hot reload as a follow-up, not part of the first configuration API. diff --git a/docs/reference.dox b/docs/reference.dox index 8d89a37e..ac7265af 100644 --- a/docs/reference.dox +++ b/docs/reference.dox @@ -235,8 +235,9 @@ storage should prefer the shared `LOGIT_READ_*` and callback macros above. File-based backends also expose persisted-file access through `LOGIT_LIST_LOG_FILES(index)`, `LOGIT_READ_LOG_FILE(index, path)`, and `LOGIT_READ_LOG_FILES(index, paths)`. These helpers return only what has -already reached disk; they do not drain async queues, and compressed rotated -files are listed as metadata-only artifacts in v1. Use `MemoryLogger` for +already reached disk; they do not drain async queues. Rotated `.gz` and `.zst` +files are decompressed when the corresponding feature is enabled; otherwise +their read result has `ok == false`. Use `MemoryLogger` for near-real-time snapshots and the file APIs for operational reads of persisted daily logs. diff --git a/include/logit_cpp/logit/detail/CompressionUtils.hpp b/include/logit_cpp/logit/detail/CompressionUtils.hpp index fea8621b..308cb2a4 100644 --- a/include/logit_cpp/logit/detail/CompressionUtils.hpp +++ b/include/logit_cpp/logit/detail/CompressionUtils.hpp @@ -78,6 +78,7 @@ inline bool compress_string_gzip(const std::string& input, std::string& output, /// \param[out] output Decompressed result (valid only on success). /// \return true on success, false if zlib is unavailable or decompression fails. inline bool decompress_string_gzip(const std::string& input, std::string& output) { + output.clear(); #if defined(LOGIT_HAS_ZLIB) z_stream zs; zs.zalloc = Z_NULL; @@ -93,8 +94,6 @@ inline bool decompress_string_gzip(const std::string& input, std::string& output return false; } - output.clear(); - int ret = Z_OK; std::size_t offset = 0; const std::size_t buf_size = 32768; @@ -107,6 +106,7 @@ inline bool decompress_string_gzip(const std::string& input, std::string& output ret = inflate(&zs, Z_NO_FLUSH); if (ret == Z_STREAM_ERROR || ret == Z_DATA_ERROR || ret == Z_MEM_ERROR) { inflateEnd(&zs); + output.clear(); return false; } @@ -159,6 +159,7 @@ inline bool compress_string_zstd(const std::string& input, std::string& output, /// \param[out] output Decompressed result (valid only on success). /// \return true on success, false if zstd is unavailable or decompression fails. inline bool decompress_string_zstd(const std::string& input, std::string& output) { + output.clear(); #if defined(LOGIT_HAS_ZSTD) std::size_t const d_size = ZSTD_getFrameContentSize(input.data(), input.size()); if (d_size == ZSTD_CONTENTSIZE_ERROR || d_size == ZSTD_CONTENTSIZE_UNKNOWN) { @@ -183,6 +184,27 @@ inline bool decompress_string_zstd(const std::string& input, std::string& output #endif } +/// \brief Decompress a file payload based on its compressed suffix. +/// \param filename File name ending in `.gz` or `.zst`. +/// \param input Compressed file bytes. +/// \param[out] output Decompressed result (valid only on success). +/// \return true when the suffix is supported and decompression succeeds. +inline bool decompress_string_by_suffix( + const std::string& filename, + const std::string& input, + std::string& output) { + if (filename.size() >= 3 && + filename.compare(filename.size() - 3, 3, ".gz") == 0) { + return decompress_string_gzip(input, output); + } + if (filename.size() >= 4 && + filename.compare(filename.size() - 4, 4, ".zst") == 0) { + return decompress_string_zstd(input, output); + } + output.clear(); + return false; +} + } // namespace detail } // namespace logit diff --git a/include/logit_cpp/logit/loggers.hpp b/include/logit_cpp/logit/loggers.hpp index f4740bd3..4e0ff89e 100644 --- a/include/logit_cpp/logit/loggers.hpp +++ b/include/logit_cpp/logit/loggers.hpp @@ -24,6 +24,7 @@ namespace logit { #ifndef __EMSCRIPTEN__ #include "detail/CompressionWorker.hpp" #endif +#include "detail/CompressionUtils.hpp" #include #include @@ -61,7 +62,6 @@ namespace logit { #endif #ifdef LOGIT_WITH_MDBX -#include "detail/CompressionUtils.hpp" #include "detail/MdbxByteIO.hpp" #include "detail/MdbxKeyUtils.hpp" #include "detail/MdbxProcessId.hpp" diff --git a/include/logit_cpp/logit/loggers/FileLogger.hpp b/include/logit_cpp/logit/loggers/FileLogger.hpp index c32a36e6..50e3dd89 100644 --- a/include/logit_cpp/logit/loggers/FileLogger.hpp +++ b/include/logit_cpp/logit/loggers/FileLogger.hpp @@ -419,7 +419,8 @@ namespace logit { /// \details This API reads only the already persisted file contents and /// does not wait for pending async writes. /// \param path Full path returned by `list_log_files()`. - /// \return Read result. Compressed files are listed but unreadable in v1. + /// \return Read result. Rotated `.gz` and `.zst` files are decompressed + /// when the corresponding feature is enabled. LogFileReadResult read_log_file(const std::string& path) const override { const std::vector files = list_log_files(); for (size_t i = 0; i < files.size(); ++i) { @@ -672,11 +673,6 @@ namespace logit { LogFileReadResult result; result.file = info; - if (info.is_compressed) { - result.ok = false; - return result; - } - if (info.is_current) { std::lock_guard lock(m_mutex); if (m_file.is_open()) { @@ -684,7 +680,18 @@ namespace logit { } } - result.ok = read_plain_file(info.path, result.content); + std::string file_bytes; + if (!read_plain_file(info.path, file_bytes)) { + result.ok = false; + return result; + } + if (info.is_compressed) { + result.ok = detail::decompress_string_by_suffix( + info.path, file_bytes, result.content); + } else { + result.content = std::move(file_bytes); + result.ok = true; + } return result; } diff --git a/include/logit_cpp/logit/loggers/UniqueFileLogger.hpp b/include/logit_cpp/logit/loggers/UniqueFileLogger.hpp index 7a1d20a3..f23307f6 100644 --- a/include/logit_cpp/logit/loggers/UniqueFileLogger.hpp +++ b/include/logit_cpp/logit/loggers/UniqueFileLogger.hpp @@ -595,12 +595,18 @@ namespace logit { LogFileReadResult read_log_file_from_info(const LogFileInfo& info) const { LogFileReadResult result; result.file = info; - if (info.is_compressed) { + std::string file_bytes; + if (!read_plain_file(info.path, file_bytes)) { result.ok = false; return result; } - - result.ok = read_plain_file(info.path, result.content); + if (info.is_compressed) { + result.ok = detail::decompress_string_by_suffix( + info.path, file_bytes, result.content); + } else { + result.content = std::move(file_bytes); + result.ok = true; + } return result; } @@ -880,7 +886,8 @@ namespace logit { /// \brief Reads one persisted log file owned by this backend. /// \param path Full path returned by `list_log_files()`. - /// \return Read result. Compressed files are listed but unreadable in v1. + /// \return Read result. Rotated `.gz` and `.zst` files are decompressed + /// when the corresponding feature is enabled. LogFileReadResult read_log_file(const std::string& path) const override { const std::vector files = list_log_files(); for (size_t i = 0; i < files.size(); ++i) { diff --git a/tests/file_logger_gzip_compression_test.cpp b/tests/file_logger_gzip_compression_test.cpp index 6bba9614..24f76225 100644 --- a/tests/file_logger_gzip_compression_test.cpp +++ b/tests/file_logger_gzip_compression_test.cpp @@ -20,13 +20,20 @@ int main() { LOGIT_INFO(msg); LOGIT_WAIT(); std::string current = LOGIT_GET_LAST_FILE_PATH(0); - LOGIT_SHUTDOWN(); std::string rotated = current; size_t pos = rotated.rfind(".log"); rotated.insert(pos, ".001"); rotated += ".gz"; + const logit::LogFileReadResult read = LOGIT_READ_LOG_FILE(0, rotated); + if (!read.ok || read.content.find(msg) == std::string::npos) return 1; + const std::vector requested = {rotated}; + const std::vector read_many = + LOGIT_READ_LOG_FILES(0, requested); + if (read_many.size() != 1 || !read_many[0].ok || + read_many[0].content.find(msg) == std::string::npos) return 1; + gzFile gz = gzopen(rotated.c_str(), "rb"); if (!gz) return 1; char buf[128]; @@ -34,7 +41,9 @@ int main() { int n; while ((n = gzread(gz, buf, sizeof(buf))) > 0) out.append(buf, n); gzclose(gz); - return out.find(msg) != std::string::npos ? 0 : 1; + const bool ok = out.find(msg) != std::string::npos; + LOGIT_SHUTDOWN(); + return ok ? 0 : 1; } #else int main() { return 0; } diff --git a/tests/file_logger_zstd_compression_test.cpp b/tests/file_logger_zstd_compression_test.cpp index 817aec73..b480f309 100644 --- a/tests/file_logger_zstd_compression_test.cpp +++ b/tests/file_logger_zstd_compression_test.cpp @@ -23,13 +23,20 @@ int main() { LOGIT_INFO(msg); LOGIT_WAIT(); std::string current = LOGIT_GET_LAST_FILE_PATH(0); - LOGIT_SHUTDOWN(); std::string rotated = current; size_t pos = rotated.rfind(".log"); rotated.insert(pos, ".001"); rotated += ".zst"; + const logit::LogFileReadResult read = LOGIT_READ_LOG_FILE(0, rotated); + if (!read.ok || read.content.find(msg) == std::string::npos) return 1; + const std::vector requested = {rotated}; + const std::vector read_many = + LOGIT_READ_LOG_FILES(0, requested); + if (read_many.size() != 1 || !read_many[0].ok || + read_many[0].content.find(msg) == std::string::npos) return 1; + std::ifstream in(rotated.c_str(), std::ios::binary | std::ios::ate); if (!in) return 1; std::streamsize size = in.tellg(); @@ -43,7 +50,9 @@ int main() { size_t ret = ZSTD_decompress(decompressed.data(), raw_size, compressed.data(), compressed.size()); if (ZSTD_isError(ret)) return 1; std::string out(decompressed.data(), ret); - return out.find(msg) != std::string::npos ? 0 : 1; + const bool ok = out.find(msg) != std::string::npos; + LOGIT_SHUTDOWN(); + return ok ? 0 : 1; } #else int main() { return 0; } diff --git a/tests/unique_file_logger_file_api_test.cpp b/tests/unique_file_logger_file_api_test.cpp index 0510696c..ac5cdf4f 100644 --- a/tests/unique_file_logger_file_api_test.cpp +++ b/tests/unique_file_logger_file_api_test.cpp @@ -69,7 +69,17 @@ int main() { } compressed_path = plain_files[0].path + ".gz"; std::ofstream gz(compressed_path.c_str(), std::ios_base::binary); + const std::string payload = "unique-compressed-payload"; +#if defined(LOGIT_HAS_ZLIB) + std::string compressed; + if (!logit::detail::compress_string_gzip(payload, compressed, 1)) { + return 1; + } + gz.write(compressed.data(), static_cast(compressed.size())); +#else gz << "compressed-placeholder"; +#endif + gz.close(); } const std::vector files = LOGIT_LIST_LOG_FILES(0); @@ -108,7 +118,11 @@ int main() { } const logit::LogFileReadResult compressed_read = LOGIT_READ_LOG_FILE(0, compressed_path); +#if defined(LOGIT_HAS_ZLIB) + if (!compressed_read.ok || compressed_read.content != "unique-compressed-payload") { +#else if (compressed_read.ok || !compressed_read.content.empty()) { +#endif return 1; } @@ -119,7 +133,12 @@ int main() { return 1; } if (!same_file_name(read_many[0].file.path, latest_path) || !read_many[0].ok || - !same_file_name(read_many[1].file.path, compressed_path) || read_many[1].ok) { + !same_file_name(read_many[1].file.path, compressed_path) || +#if defined(LOGIT_HAS_ZLIB) + !read_many[1].ok || read_many[1].content != "unique-compressed-payload") { +#else + read_many[1].ok) { +#endif return 1; }