From e3cb97657f1d4135f54e18c80980e7b3e7f3cbcb Mon Sep 17 00:00:00 2001 From: Chad Attermann Date: Sun, 21 Jun 2026 09:39:44 -0600 Subject: [PATCH 1/3] Implemented schema hashing to support client caching --- src/microReticulum/Provisioning/Ops.h | 1 + .../Provisioning/Provisioning.cpp | 131 +++++++++++------- .../Provisioning/Provisioning.h | 6 + 3 files changed, 85 insertions(+), 53 deletions(-) diff --git a/src/microReticulum/Provisioning/Ops.h b/src/microReticulum/Provisioning/Ops.h index 149b0cd..763d76c 100644 --- a/src/microReticulum/Provisioning/Ops.h +++ b/src/microReticulum/Provisioning/Ops.h @@ -79,6 +79,7 @@ namespace RNS { namespace Provisioning { constexpr uint16_t FirmwareVersion = 1; constexpr uint16_t SchemaVersion = 2; constexpr uint16_t NeedsRebootInfo = 3; + constexpr uint16_t SchemaHash = 4; // CRC32 of serialized GetSchema response bytes // GetCapabilities constexpr uint16_t Namespaces = 1; diff --git a/src/microReticulum/Provisioning/Provisioning.cpp b/src/microReticulum/Provisioning/Provisioning.cpp index 7b41c86..eb54fab 100644 --- a/src/microReticulum/Provisioning/Provisioning.cpp +++ b/src/microReticulum/Provisioning/Provisioning.cpp @@ -16,6 +16,7 @@ #include "Codec.h" #include "../Log.h" +#include "../Utilities/Crc.h" #include @@ -24,6 +25,12 @@ namespace RNS { namespace Provisioning { // Defined in BuiltinNamespaces.cpp. void register_builtin_namespaces(Provisioner& p); + // Forward declaration — definition lives next to op_get_schema below so + // the schema serialization logic stays grouped. Called from begin() to + // compute the boot-time schema hash and from op_get_schema for the wire + // path so the hash matches the bytes clients actually receive. + static void pack_schema_payload(MsgPack::Packer& p, const Registry& registry); + // --------------------------------------------------------------------- // Singleton // --------------------------------------------------------------------- @@ -60,6 +67,16 @@ namespace RNS { namespace Provisioning { (void)storage_root; #endif _needs_reboot = false; + // Hash the schema payload as it will go out on the wire. Computed + // once here after registration is complete so GetInfo can report + // it as a cache key — clients keying their local schema cache by + // this hash skip a full GetSchema fetch when the device's schema + // has not changed. + { + MsgPack::Packer hp; + pack_schema_payload(hp, _registry); + _schema_hash = Utilities::Crc::crc32(0, (const uint8_t*)hp.data(), hp.size()); + } _started = true; } @@ -377,10 +394,11 @@ namespace RNS { namespace Provisioning { Bytes Provisioner::op_get_info(seq_t seq) { return pack_response((opid_t)Op::GetInfo, seq, [&](MsgPack::Packer& p) { - p.serialize(MsgPack::map_size_t(3)); + p.serialize(MsgPack::map_size_t(4)); p.serialize((uint16_t)Key::FirmwareVersion); p.serialize("microReticulum"); p.serialize((uint16_t)Key::SchemaVersion); p.serialize((nid_t)Provisioner::SCHEMA_VERSION); p.serialize((uint16_t)Key::NeedsRebootInfo); p.serialize((bool)_needs_reboot); + p.serialize((uint16_t)Key::SchemaHash); p.serialize((uint32_t)_schema_hash); }); } @@ -440,64 +458,71 @@ namespace RNS { namespace Provisioning { return n; } - Bytes Provisioner::op_get_schema(seq_t seq) { - return pack_response((opid_t)Op::GetSchema, seq, [&](MsgPack::Packer& p) { - const auto& nss = _registry.namespaces(); - p.serialize(MsgPack::arr_size_t(nss.size())); - for (const auto& ns_ptr : nss) { - const Namespace& ns = *ns_ptr; - // Each namespace is [id, name, parent_id_or_zero, [field-maps]]. - // parent_id of 0 means root (no parent). Schema v2 layout — - // v1 clients reading the first three elements still parse - // the rest of the response correctly. - p.serialize(MsgPack::arr_size_t(4)); - p.serialize((nid_t)ns.id()); - p.serialize(ns.name().c_str()); - p.serialize((nid_t)ns.parent_id()); - const auto& fields = ns.fields(); - p.serialize(MsgPack::arr_size_t(fields.size())); - for (const Field& f : fields) { - p.serialize(MsgPack::map_size_t(schema_field_entries(f))); - p.serialize((uint16_t)Key::FieldId); p.serialize((fid_t)f.id); - p.serialize((uint16_t)Key::FieldName); p.serialize(f.name.c_str()); - p.serialize((uint16_t)Key::FieldType); p.serialize((uint8_t)f.type); - p.serialize((uint16_t)Key::FieldFlags); p.serialize((fflags_t)f.flags); - p.serialize((uint16_t)Key::FieldDefault); pack_field_default(p, f); - if (f.type == Type::Int && f.constraint.has_range) { - p.serialize((uint16_t)Key::FieldMinI); p.serialize((fint_t)f.constraint.imin); - p.serialize((uint16_t)Key::FieldMaxI); p.serialize((fint_t)f.constraint.imax); - } - if (f.type == Type::Float && f.constraint.has_range) { - p.serialize((uint16_t)Key::FieldMinF); p.serialize((double)f.constraint.fmin); - p.serialize((uint16_t)Key::FieldMaxF); p.serialize((double)f.constraint.fmax); + // Packs the GetSchema payload (just the namespaces array) onto `p`. + // Shared by op_get_schema() and the boot-time CRC32 hashing path so + // the hash is computed over the exact bytes clients receive. + static void pack_schema_payload(MsgPack::Packer& p, const Registry& registry) { + const auto& nss = registry.namespaces(); + p.serialize(MsgPack::arr_size_t(nss.size())); + for (const auto& ns_ptr : nss) { + const Namespace& ns = *ns_ptr; + // Each namespace is [id, name, parent_id_or_zero, [field-maps]]. + // parent_id of 0 means root (no parent). Schema v2 layout — + // v1 clients reading the first three elements still parse + // the rest of the response correctly. + p.serialize(MsgPack::arr_size_t(4)); + p.serialize((nid_t)ns.id()); + p.serialize(ns.name().c_str()); + p.serialize((nid_t)ns.parent_id()); + const auto& fields = ns.fields(); + p.serialize(MsgPack::arr_size_t(fields.size())); + for (const Field& f : fields) { + p.serialize(MsgPack::map_size_t(schema_field_entries(f))); + p.serialize((uint16_t)Key::FieldId); p.serialize((fid_t)f.id); + p.serialize((uint16_t)Key::FieldName); p.serialize(f.name.c_str()); + p.serialize((uint16_t)Key::FieldType); p.serialize((uint8_t)f.type); + p.serialize((uint16_t)Key::FieldFlags); p.serialize((fflags_t)f.flags); + p.serialize((uint16_t)Key::FieldDefault); pack_field_default(p, f); + if (f.type == Type::Int && f.constraint.has_range) { + p.serialize((uint16_t)Key::FieldMinI); p.serialize((fint_t)f.constraint.imin); + p.serialize((uint16_t)Key::FieldMaxI); p.serialize((fint_t)f.constraint.imax); + } + if (f.type == Type::Float && f.constraint.has_range) { + p.serialize((uint16_t)Key::FieldMinF); p.serialize((double)f.constraint.fmin); + p.serialize((uint16_t)Key::FieldMaxF); p.serialize((double)f.constraint.fmax); + } + if ((f.type == Type::String || f.type == Type::Bytes) && f.constraint.max_len > 0) { + p.serialize((uint16_t)Key::FieldMaxLen); p.serialize((uint64_t)f.constraint.max_len); + } + if (f.type == Type::Enum) { + if (!f.constraint.enum_values.empty()) { + p.serialize((uint16_t)Key::FieldEnumValues); + p.serialize(MsgPack::arr_size_t(f.constraint.enum_values.size())); + for (fenum_t v : f.constraint.enum_values) p.serialize(v); } - if ((f.type == Type::String || f.type == Type::Bytes) && f.constraint.max_len > 0) { - p.serialize((uint16_t)Key::FieldMaxLen); p.serialize((uint64_t)f.constraint.max_len); + if (!f.constraint.enum_labels.empty()) { + p.serialize((uint16_t)Key::FieldEnumLabels); + p.serialize(MsgPack::arr_size_t(f.constraint.enum_labels.size())); + for (const auto& s : f.constraint.enum_labels) p.serialize(s.c_str()); } - if (f.type == Type::Enum) { - if (!f.constraint.enum_values.empty()) { - p.serialize((uint16_t)Key::FieldEnumValues); - p.serialize(MsgPack::arr_size_t(f.constraint.enum_values.size())); - for (fenum_t v : f.constraint.enum_values) p.serialize(v); - } - if (!f.constraint.enum_labels.empty()) { - p.serialize((uint16_t)Key::FieldEnumLabels); - p.serialize(MsgPack::arr_size_t(f.constraint.enum_labels.size())); - for (const auto& s : f.constraint.enum_labels) p.serialize(s.c_str()); - } + } + if (f.type == Type::BytesList) { + if (f.constraint.element_size > 0) { + p.serialize((uint16_t)Key::FieldElementSize); + p.serialize((uint64_t)f.constraint.element_size); } - if (f.type == Type::BytesList) { - if (f.constraint.element_size > 0) { - p.serialize((uint16_t)Key::FieldElementSize); - p.serialize((uint64_t)f.constraint.element_size); - } - if (f.constraint.max_count > 0) { - p.serialize((uint16_t)Key::FieldMaxCount); - p.serialize((uint64_t)f.constraint.max_count); - } + if (f.constraint.max_count > 0) { + p.serialize((uint16_t)Key::FieldMaxCount); + p.serialize((uint64_t)f.constraint.max_count); } } } + } + } + + Bytes Provisioner::op_get_schema(seq_t seq) { + return pack_response((opid_t)Op::GetSchema, seq, [&](MsgPack::Packer& p) { + pack_schema_payload(p, _registry); }); } diff --git a/src/microReticulum/Provisioning/Provisioning.h b/src/microReticulum/Provisioning/Provisioning.h index 391560b..e431e50 100644 --- a/src/microReticulum/Provisioning/Provisioning.h +++ b/src/microReticulum/Provisioning/Provisioning.h @@ -246,6 +246,11 @@ namespace RNS { namespace Provisioning { // stop after reading the first three elements remain compatible. static constexpr nid_t SCHEMA_VERSION = 2; + // CRC32 of the serialized GetSchema payload, computed once at the + // end of begin(). Surfaced in GetInfo as Key::SchemaHash so clients + // can cache the schema by hash and skip re-fetching when unchanged. + uint32_t schema_hash() const { return _schema_hash; } + private: Provisioner() = default; Provisioner(const Provisioner&) = delete; @@ -255,6 +260,7 @@ namespace RNS { namespace Provisioning { std::unique_ptr _storage; bool _started = false; bool _needs_reboot = false; + uint32_t _schema_hash = 0; RebootRequiredCallback _on_reboot_required; RebootCallback _on_reboot; FactoryResetCallback _on_factory_reset; From 6ebf17ec700e4d0675d5c80c016211ae7bedca48 Mon Sep 17 00:00:00 2001 From: Chad Attermann Date: Mon, 22 Jun 2026 09:12:22 -0600 Subject: [PATCH 2/3] Added support for compression to Provisioning --- CMakeLists.txt | 9 +- library.json | 6 +- library.properties | 2 +- platformio.ini | 1 + src/microReticulum/Provisioning/Ops.h | 12 ++ .../Provisioning/Provisioning.cpp | 137 ++++++++++++++---- .../Provisioning/Provisioning.h | 15 +- src/microReticulum/Provisioning/Storage.cpp | 31 ++-- src/microReticulum/Provisioning/Storage.h | 10 +- src/microReticulum/Utilities/Compress.cpp | 136 +++++++++++++++++ src/microReticulum/Utilities/Compress.h | 47 ++++++ 11 files changed, 345 insertions(+), 61 deletions(-) create mode 100644 src/microReticulum/Utilities/Compress.cpp create mode 100644 src/microReticulum/Utilities/Compress.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 60d350e..48fe828 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -35,6 +35,7 @@ set(RNS_HEAP_POOL_BUFFER_SIZE "" CACHE STRING "TLSF heap pool size (bytes)") set(RNS_ARDUINOJSON_SOURCE_DIR "" CACHE PATH "Use local ArduinoJson checkout instead of FetchContent") set(RNS_MSGPACK_SOURCE_DIR "" CACHE PATH "Use local MsgPack checkout instead of FetchContent") set(RNS_CRYPTO_SOURCE_DIR "" CACHE PATH "Use local Crypto checkout instead of FetchContent") +set(RNS_HEATSHRINK_SOURCE_DIR "" CACHE PATH "Use local Heatshrink checkout instead of FetchContent") set(RNS_MICROSTORE_SOURCE_DIR "" CACHE PATH "Use local microStore checkout instead of FetchContent") set(RNS_ARXCONTAINER_SOURCE_DIR "" CACHE PATH "Use local ArxContainer checkout instead of FetchContent") set(RNS_ARXTYPETRAITS_SOURCE_DIR "" CACHE PATH "Use local ArxTypeTraits checkout instead of FetchContent") @@ -65,10 +66,11 @@ rns_fetch(msgpack RNS_MSGPACK_SOURCE_DIR https://github.com/hideak rns_fetch(arxcontainer RNS_ARXCONTAINER_SOURCE_DIR https://github.com/hideakitai/ArxContainer.git v0.7.0) rns_fetch(arxtypetraits RNS_ARXTYPETRAITS_SOURCE_DIR https://github.com/hideakitai/ArxTypeTraits.git v0.3.2) rns_fetch(debuglog RNS_DEBUGLOG_SOURCE_DIR https://github.com/hideakitai/DebugLog.git v0.8.4) +rns_fetch(heatshrink RNS_HEATSHRINK_SOURCE_DIR https://github.com/atomicobject/heatshrink.git v0.4.1) # Crypto and microStore have no upstream tags suitable for pinning, so pin commit SHAs. # Refresh by running `git ls-remote HEAD` and updating the hex below. -rns_fetch(crypto RNS_CRYPTO_SOURCE_DIR https://github.com/attermann/Crypto.git 984dc891330986c302a86c4e312d4f5abcc28359) -rns_fetch(microstore RNS_MICROSTORE_SOURCE_DIR https://github.com/attermann/microStore.git c5fb69d68229e684c7fbd17692a67ae8193b84e2) +rns_fetch(crypto RNS_CRYPTO_SOURCE_DIR https://github.com/attermann/Crypto.git) +rns_fetch(microstore RNS_MICROSTORE_SOURCE_DIR https://github.com/attermann/microStore.git) # ----------------------------------------------------------------------------- # Third-party targets that don't ship CMakeLists.txt @@ -88,6 +90,8 @@ add_library(ArxTypeTraitsHeaders INTERFACE) target_include_directories(ArxTypeTraitsHeaders INTERFACE "${arxtypetraits_SOURCE_DIR}") add_library(DebugLogHeaders INTERFACE) target_include_directories(DebugLogHeaders INTERFACE "${debuglog_SOURCE_DIR}") +add_library(HeatshrinkHeaders INTERFACE) +target_include_directories(HeatshrinkHeaders INTERFACE "${heatshrink_SOURCE_DIR}") add_library(microStoreHeaders INTERFACE) target_include_directories(microStoreHeaders INTERFACE "${microstore_SOURCE_DIR}/include") @@ -155,6 +159,7 @@ target_link_libraries(microReticulum ArxContainerHeaders ArxTypeTraitsHeaders DebugLogHeaders + HeatshrinkHeaders microStoreHeaders CryptoLib ) diff --git a/library.json b/library.json index 2b1b56e..382d178 100644 --- a/library.json +++ b/library.json @@ -1,6 +1,6 @@ { "name": "microReticulum", - "version": "0.4.1", + "version": "0.4.2", "description": "C++ port of the Reticulum Network Stack — a cryptographic mesh networking library that runs on both 32-bit+ microcontrollers (ESP32, nRF52, LoRa boards) and native platforms (Linux, macOS), with PlatformIO and CMake build support.", "keywords": "reticulum, rns, mesh, embedded, mcu, esp32, nrf52, native", "repository": @@ -27,6 +27,10 @@ "name": "MsgPack", "version": "~0.4.2" }, + { + "name": "heatshrink", + "url": "https://github.com/atomicobject/heatshrink.git#v0.4.1" + }, { "name": "Crypto", "url": "https://github.com/attermann/Crypto.git" diff --git a/library.properties b/library.properties index 6691d27..a96ed96 100644 --- a/library.properties +++ b/library.properties @@ -1,5 +1,5 @@ name=microReticulum -version=0.4.1 +version=0.4.2 author=Chad Attermann maintainer=Chad Attermann sentence=C++ port of the Reticulum Network Stack diff --git a/platformio.ini b/platformio.ini index 26e3edf..c98a133 100644 --- a/platformio.ini +++ b/platformio.ini @@ -47,6 +47,7 @@ build_flags = lib_deps = ArduinoJson@^7.4.2 MsgPack@^0.4.2 + https://github.com/atomicobject/heatshrink.git#v0.4.1 https://github.com/attermann/Crypto.git test_framework = unity ; Following property enables Unity integration testing diff --git a/src/microReticulum/Provisioning/Ops.h b/src/microReticulum/Provisioning/Ops.h index 763d76c..3c425fc 100644 --- a/src/microReticulum/Provisioning/Ops.h +++ b/src/microReticulum/Provisioning/Ops.h @@ -104,6 +104,18 @@ namespace RNS { namespace Provisioning { constexpr uint16_t FieldDefault = 12; constexpr uint16_t FieldElementSize = 13; // BytesList: required size per entry constexpr uint16_t FieldMaxCount = 14; // BytesList: max number of entries + + // Per-request compression negotiation. Clients set ReqCompress=true + // in the request payload map to ask the server to compress the + // response. When the server complies, the response payload becomes + // a single-entry map { CompressedPayload: } whose value is + // heatshrink-compressed bytes that decompress to the original + // MsgPack-encoded payload. Servers always pick the smaller wire + // form and may send an uncompressed payload even when compression + // was requested (tiny payloads where the wrapper overhead exceeds + // the savings). + constexpr uint16_t ReqCompress = 100; + constexpr uint16_t CompressedPayload = 101; } } } diff --git a/src/microReticulum/Provisioning/Provisioning.cpp b/src/microReticulum/Provisioning/Provisioning.cpp index eb54fab..e8c83e4 100644 --- a/src/microReticulum/Provisioning/Provisioning.cpp +++ b/src/microReticulum/Provisioning/Provisioning.cpp @@ -17,7 +17,9 @@ #include "../Log.h" #include "../Utilities/Crc.h" +#include "../Utilities/Compress.h" +#include #include namespace RNS { namespace Provisioning { @@ -338,14 +340,54 @@ namespace RNS { namespace Provisioning { // [ op_id (uint), seq (uint), payload ] // where payload is op-specific (often a map; nil if not used). - static Bytes pack_response(opid_t op_id, seq_t seq, const std::function& pack_payload) { - MsgPack::Packer packer; - packer.serialize(MsgPack::arr_size_t(3)); - packer.serialize((opid_t)op_id); - packer.serialize((seq_t)seq); - if (pack_payload) pack_payload(packer); - else { MsgPack::object::nil_t n; packer.serialize(n); } - return Bytes(packer.data(), packer.size()); + static Bytes pack_response(opid_t op_id, seq_t seq, bool compress, const std::function& pack_payload) { + // Build the payload portion into a scratch packer first so we can + // optionally compress it before stitching it into the envelope. + MsgPack::Packer payload_pkt; + if (pack_payload) pack_payload(payload_pkt); + else { MsgPack::object::nil_t n; payload_pkt.serialize(n); } + + // Envelope prefix: [arr-size 3, op_id, seq]. The third slot is filled + // below — either with the {CompressedPayload: bin} wrapper or by + // appending the uncompressed payload bytes verbatim. + MsgPack::Packer envelope; + envelope.serialize(MsgPack::arr_size_t(3)); + envelope.serialize((opid_t)op_id); + envelope.serialize((seq_t)seq); + + if (compress) { + Bytes compressed = Utilities::Compress::encode( + (const uint8_t*)payload_pkt.data(), payload_pkt.size()); + // Wrapper bytes: map-of-1 header (1 B) + uint16 key (3 B) + + // bin header (2 B for <=255, 3 B otherwise) = up to 7 B. Use 8 + // as a safe upper bound. Only emit the compressed wrapper if + // it nets bytes saved over the raw payload — otherwise the + // "compressed" envelope would actually be larger on the wire. + constexpr size_t WRAPPER_OVERHEAD = 8; + if (!compressed.empty() && + compressed.size() + WRAPPER_OVERHEAD < payload_pkt.size()) + { + envelope.serialize(MsgPack::map_size_t(1)); + envelope.serialize((uint16_t)Key::CompressedPayload); + MsgPack::bin_t bin; + bin.resize(compressed.size()); + memcpy(bin.data(), compressed.data(), compressed.size()); + envelope.serialize(bin); + return Bytes(envelope.data(), envelope.size()); + } + // Fall through: compression unprofitable, send raw payload. + } + + // Uncompressed path: append the scratch payload bytes verbatim onto + // the envelope prefix. Same wire shape as before this refactor. + Bytes out((const uint8_t*)envelope.data(), envelope.size()); + out.append((const uint8_t*)payload_pkt.data(), payload_pkt.size()); + return out; + } + + // Convenience overload: no-compress callers (Error, Ack, internal helpers). + static inline Bytes pack_response(opid_t op_id, seq_t seq, const std::function& pack_payload) { + return pack_response(op_id, seq, false, pack_payload); } Bytes Provisioner::encode_error(opid_t op_id, seq_t seq, ErrorCode code, const char* msg) { @@ -392,8 +434,32 @@ namespace RNS { namespace Provisioning { } } - Bytes Provisioner::op_get_info(seq_t seq) { - return pack_response((opid_t)Op::GetInfo, seq, [&](MsgPack::Packer& p) { + // Parse a request payload map for the ReqCompress flag. Used by op + // handlers whose request payload carries no other keys (GetSchema, + // GetInfo, GetCapabilities, FactoryReset, Reboot). Ops with their own + // payload schema (GetState, SetState, Commit, Discard) handle ReqCompress + // inline alongside their own keys. Consumes the payload value either way. + static bool parse_compress_only(void* unpacker_v) { + MsgPack::Unpacker* up = (MsgPack::Unpacker*)unpacker_v; + if (!up) return false; + if (!up->isMap()) { skip_value(*up); return false; } + const size_t n = up->unpackMapSize(); + bool compress = false; + for (size_t i = 0; i < n; ++i) { + nid_t key; + if (!read_uint_key(*up, key)) { skip_value(*up); continue; } + if (key == Key::ReqCompress) { + if (up->isBool()) { bool v = false; up->deserialize(v); compress = v; } + else skip_value(*up); + } + else skip_value(*up); + } + return compress; + } + + Bytes Provisioner::op_get_info(seq_t seq, void* unpacker_v) { + const bool compress = parse_compress_only(unpacker_v); + return pack_response((opid_t)Op::GetInfo, seq, compress, [&](MsgPack::Packer& p) { p.serialize(MsgPack::map_size_t(4)); p.serialize((uint16_t)Key::FirmwareVersion); p.serialize("microReticulum"); p.serialize((uint16_t)Key::SchemaVersion); p.serialize((nid_t)Provisioner::SCHEMA_VERSION); @@ -402,8 +468,9 @@ namespace RNS { namespace Provisioning { }); } - Bytes Provisioner::op_get_capabilities(seq_t seq) { - return pack_response((opid_t)Op::GetCapabilities, seq, [&](MsgPack::Packer& p) { + Bytes Provisioner::op_get_capabilities(seq_t seq, void* unpacker_v) { + const bool compress = parse_compress_only(unpacker_v); + return pack_response((opid_t)Op::GetCapabilities, seq, compress, [&](MsgPack::Packer& p) { const auto& nss = _registry.namespaces(); p.serialize(MsgPack::arr_size_t(nss.size())); for (const auto& ns_ptr : nss) { @@ -520,8 +587,9 @@ namespace RNS { namespace Provisioning { } } - Bytes Provisioner::op_get_schema(seq_t seq) { - return pack_response((opid_t)Op::GetSchema, seq, [&](MsgPack::Packer& p) { + Bytes Provisioner::op_get_schema(seq_t seq, void* unpacker_v) { + const bool compress = parse_compress_only(unpacker_v); + return pack_response((opid_t)Op::GetSchema, seq, compress, [&](MsgPack::Packer& p) { pack_schema_payload(p, _registry); }); } @@ -536,7 +604,8 @@ namespace RNS { namespace Provisioning { std::unordered_set ns_filter; bool has_filter = false; bool pending = false; - // Optional payload map: {1: [ns_filter], 2: pending} + bool compress = false; + // Optional payload map: {1: [ns_filter], 2: pending, 100: compress} if (up && up->isMap()) { const size_t n = up->unpackMapSize(); for (size_t i = 0; i < n; ++i) { @@ -560,12 +629,16 @@ namespace RNS { namespace Provisioning { if (up->isBool()) up->deserialize(pending); else skip_value(*up); } + else if (key == Key::ReqCompress) { + if (up->isBool()) up->deserialize(compress); + else skip_value(*up); + } else skip_value(*up); } } else if (up) skip_value(*up); - return pack_response((opid_t)Op::GetState, seq, [&](MsgPack::Packer& p) { + return pack_response((opid_t)Op::GetState, seq, compress, [&](MsgPack::Packer& p) { std::vector ns_list; for (const auto& ns_ptr : _registry.namespaces()) { if (has_filter && ns_filter.count(ns_ptr->id()) == 0) continue; @@ -779,15 +852,19 @@ namespace RNS { namespace Provisioning { }); } - Bytes Provisioner::op_factory_reset(seq_t seq) { + Bytes Provisioner::op_factory_reset(seq_t seq, void* unpacker_v) { + const bool compress = parse_compress_only(unpacker_v); factory_reset(); - return pack_response((opid_t)Op::FactoryReset, seq, [&](MsgPack::Packer& p) { + return pack_response((opid_t)Op::FactoryReset, seq, compress, [&](MsgPack::Packer& p) { p.serialize(MsgPack::map_size_t(1)); p.serialize((uint16_t)Key::NeedsReboot); p.serialize((bool)_needs_reboot); }); } - Bytes Provisioner::op_reboot(seq_t seq) { + Bytes Provisioner::op_reboot(seq_t seq, void* unpacker_v) { + // Consume the payload (including any ReqCompress flag, which is moot + // here — the response is a plain Ack and would only grow if wrapped). + parse_compress_only(unpacker_v); // microReticulum performs no reboot itself; if the app registered a // callback, fire it. The ack is still returned either way so the // client can always observe success at the wire layer. @@ -836,16 +913,20 @@ namespace RNS { namespace Provisioning { // Element 3: payload (optional). May be nil. const bool has_payload = (arr_size >= 3); + // Every op handler now consumes the payload itself (so it can pick + // up the ReqCompress flag where applicable). A nullptr unpacker + // means the request had no payload element at all. + void* up_ptr = has_payload ? (void*)&up : nullptr; switch ((Op)op_id) { - case Op::GetSchema: if (has_payload) skip_value(up); return op_get_schema(seq); - case Op::GetInfo: if (has_payload) skip_value(up); return op_get_info(seq); - case Op::GetCapabilities: if (has_payload) skip_value(up); return op_get_capabilities(seq); - case Op::GetState: return op_get_state(seq, has_payload ? &up : nullptr); - case Op::SetState: return op_set_state(seq, has_payload ? &up : nullptr); - case Op::Commit: return op_commit(seq, has_payload ? &up : nullptr); - case Op::Discard: return op_discard(seq, has_payload ? &up : nullptr); - case Op::FactoryReset: if (has_payload) skip_value(up); return op_factory_reset(seq); - case Op::Reboot: if (has_payload) skip_value(up); return op_reboot(seq); + case Op::GetSchema: return op_get_schema(seq, up_ptr); + case Op::GetInfo: return op_get_info(seq, up_ptr); + case Op::GetCapabilities: return op_get_capabilities(seq, up_ptr); + case Op::GetState: return op_get_state(seq, up_ptr); + case Op::SetState: return op_set_state(seq, up_ptr); + case Op::Commit: return op_commit(seq, up_ptr); + case Op::Discard: return op_discard(seq, up_ptr); + case Op::FactoryReset: return op_factory_reset(seq, up_ptr); + case Op::Reboot: return op_reboot(seq, up_ptr); default: return encode_error(op_id, seq, ErrorCode::UnknownOp, "unrecognised op id"); } diff --git a/src/microReticulum/Provisioning/Provisioning.h b/src/microReticulum/Provisioning/Provisioning.h index e431e50..3d02714 100644 --- a/src/microReticulum/Provisioning/Provisioning.h +++ b/src/microReticulum/Provisioning/Provisioning.h @@ -291,17 +291,20 @@ namespace RNS { namespace Provisioning { // Per-op response builders. The unpacker is positioned at the // envelope's payload value (which may be nil / map / array depending - // on op). The returned Bytes is a fully framed response. + // on op), or null if the envelope had no payload element. The + // returned Bytes is a fully framed response. Each handler reads its + // payload (including the optional ReqCompress flag) and calls + // pack_response with the resolved compress decision. class Unpacker; // forward; defined in .cpp - Bytes op_get_schema(seq_t seq); - Bytes op_get_info(seq_t seq); - Bytes op_get_capabilities(seq_t seq); + Bytes op_get_schema(seq_t seq, void* unpacker); + Bytes op_get_info(seq_t seq, void* unpacker); + Bytes op_get_capabilities(seq_t seq, void* unpacker); Bytes op_get_state(seq_t seq, void* unpacker); Bytes op_set_state(seq_t seq, void* unpacker); Bytes op_commit(seq_t seq, void* unpacker); Bytes op_discard(seq_t seq, void* unpacker); - Bytes op_factory_reset(seq_t seq); - Bytes op_reboot(seq_t seq); + Bytes op_factory_reset(seq_t seq, void* unpacker); + Bytes op_reboot(seq_t seq, void* unpacker); void set_reboot_flag(bool any_reboot_applied); diff --git a/src/microReticulum/Provisioning/Storage.cpp b/src/microReticulum/Provisioning/Storage.cpp index fa4af3a..48785a3 100644 --- a/src/microReticulum/Provisioning/Storage.cpp +++ b/src/microReticulum/Provisioning/Storage.cpp @@ -21,27 +21,24 @@ namespace RNS { namespace Provisioning { - fstring_t Storage::dotted_name(const Namespace& ns) const { - fstring_t acc = ns.name(); - if (!_registry) return acc; - nid_t hop = ns.parent_id(); - // Bounded walk — registries are small. If the chain points at a - // missing namespace we stop and return whatever we have so far. - while (hop != 0) { - const Namespace* p = _registry->find(hop); - if (!p) break; - acc = p->name() + "." + acc; - hop = p->parent_id(); - } - return acc; - } - + // Persistence file names are derived from the namespace id, not the + // human-readable name, so length stays bounded ("ns65535.msgpack.tmp" = + // 19 chars) and stays valid on the strictest embedded filesystems we + // target. Namespace ids are already guaranteed unique by the registry, + // so flattening the hierarchy in the filename is safe — parent + // relationships are still recorded in the serialized payload. + // + // Note: this is a storage-format break vs the prior name-based scheme. + // Existing on-disk files written under names like "RNode General Config + // .msgpack" will not be loaded by the id-based code path; devices with + // stale config files should erase storage or accept reverting to + // defaults. fstring_t Storage::file_path(const Namespace& ns) const { - return _root + "/" + dotted_name(ns) + ".msgpack"; + return _root + "/ns" + std::to_string(ns.id()) + ".msgpack"; } fstring_t Storage::tmp_path(const Namespace& ns) const { - return _root + "/" + dotted_name(ns) + ".msgpack.tmp"; + return _root + "/ns" + std::to_string(ns.id()) + ".msgpack.tmp"; } bool Storage::ensure_directory() { diff --git a/src/microReticulum/Provisioning/Storage.h b/src/microReticulum/Provisioning/Storage.h index 3d58219..84aa84b 100644 --- a/src/microReticulum/Provisioning/Storage.h +++ b/src/microReticulum/Provisioning/Storage.h @@ -59,16 +59,14 @@ namespace RNS { namespace Provisioning { private: fstring_t _root; - const Registry* _registry; // optional; used for dotted-path filenames + const Registry* _registry; // optional; reserved for future name-based + // lookups; the file_path/tmp_path helpers + // derive names from ns.id() and do not + // need the registry at all. fstring_t file_path(const Namespace& ns) const; fstring_t tmp_path(const Namespace& ns) const; - // Builds "Parent.Child.GrandChild" by walking ns.parent_id() up the - // Registry. Falls back to ns.name() alone if _registry is null or - // a parent link is broken. - fstring_t dotted_name(const Namespace& ns) const; - bool load_namespace(Namespace& ns); }; diff --git a/src/microReticulum/Utilities/Compress.cpp b/src/microReticulum/Utilities/Compress.cpp new file mode 100644 index 0000000..f52c498 --- /dev/null +++ b/src/microReticulum/Utilities/Compress.cpp @@ -0,0 +1,136 @@ +/* + * Copyright (c) 2026 Chad Attermann + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + */ + +#include "Compress.h" + +extern "C" { +#include "heatshrink_encoder.h" +#include "heatshrink_decoder.h" +} + +namespace RNS { namespace Utilities { + + // Output scratch — sized to amortize encoder_poll / decoder_poll calls. + // Each loop iteration drains up to this many bytes from heatshrink before + // returning to feeding more input, keeping the call count bounded. + static constexpr size_t POLL_CHUNK = 128; + + Bytes Compress::encode(const uint8_t* in_data, size_t in_len) { + if (!in_data || in_len == 0) return Bytes(); + + // Encoder struct is ~3 KB with WINDOW_BITS=9; stack-local is fine + // on ESP32/nRF52 task stacks (8 KB+). If this ever migrates to a + // tighter target, switch to `new heatshrink_encoder` + delete. + heatshrink_encoder hse; + heatshrink_encoder_reset(&hse); + + // Pre-reserve worst-case-input bytes. If we ever produce more than + // in_len, compression is unprofitable — bail out so callers send the + // uncompressed payload instead of wrapping a larger blob. + Bytes out(in_len); + uint8_t scratch[POLL_CHUNK]; + + // Feed phase: sink all input into the encoder, draining output as + // the encoder's internal buffer fills. + size_t in_pos = 0; + while (in_pos < in_len) { + size_t sunk = 0; + HSE_sink_res sres = heatshrink_encoder_sink( + &hse, + const_cast(in_data + in_pos), + in_len - in_pos, + &sunk); + if (sres < 0) return Bytes(); + in_pos += sunk; + + HSE_poll_res pres; + do { + size_t produced = 0; + pres = heatshrink_encoder_poll(&hse, scratch, sizeof(scratch), &produced); + if (pres < 0) return Bytes(); + if (produced > 0) { + if (out.size() + produced >= in_len) return Bytes(); + out.append(scratch, produced); + } + } while (pres == HSER_POLL_MORE); + } + + // Finish phase: tell the encoder the input is done and drain the + // remaining bits. May still produce output across several polls. + HSE_finish_res fres = heatshrink_encoder_finish(&hse); + while (fres == HSER_FINISH_MORE) { + size_t produced = 0; + HSE_poll_res pres = heatshrink_encoder_poll(&hse, scratch, sizeof(scratch), &produced); + if (pres < 0) return Bytes(); + if (produced > 0) { + if (out.size() + produced >= in_len) return Bytes(); + out.append(scratch, produced); + } + fres = heatshrink_encoder_finish(&hse); + } + if (fres != HSER_FINISH_DONE) return Bytes(); + + return out; + } + + Bytes Compress::decode(const uint8_t* in_data, size_t in_len, size_t max_out) { + if (!in_data || in_len == 0 || max_out == 0) return Bytes(); + + // Decoder static size = 2^WINDOW_BITS + small state ≈ 1 KB. + heatshrink_decoder hsd; + heatshrink_decoder_reset(&hsd); + + Bytes out; + uint8_t scratch[POLL_CHUNK]; + + size_t in_pos = 0; + while (in_pos < in_len) { + size_t sunk = 0; + HSD_sink_res sres = heatshrink_decoder_sink( + &hsd, + const_cast(in_data + in_pos), + in_len - in_pos, + &sunk); + if (sres < 0) return Bytes(); + in_pos += sunk; + + HSD_poll_res pres; + do { + size_t produced = 0; + pres = heatshrink_decoder_poll(&hsd, scratch, sizeof(scratch), &produced); + if (pres < 0) return Bytes(); + if (produced > 0) { + if (out.size() + produced > max_out) return Bytes(); + out.append(scratch, produced); + } + } while (pres == HSDR_POLL_MORE); + } + + HSD_finish_res fres = heatshrink_decoder_finish(&hsd); + while (fres == HSDR_FINISH_MORE) { + size_t produced = 0; + HSD_poll_res pres = heatshrink_decoder_poll(&hsd, scratch, sizeof(scratch), &produced); + if (pres < 0) return Bytes(); + if (produced > 0) { + if (out.size() + produced > max_out) return Bytes(); + out.append(scratch, produced); + } + fres = heatshrink_decoder_finish(&hsd); + } + if (fres != HSDR_FINISH_DONE) return Bytes(); + + return out; + } + +} } diff --git a/src/microReticulum/Utilities/Compress.h b/src/microReticulum/Utilities/Compress.h new file mode 100644 index 0000000..8cab664 --- /dev/null +++ b/src/microReticulum/Utilities/Compress.h @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2026 Chad Attermann + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + */ + +#pragma once + +#include "../Bytes.h" + +#include +#include + +namespace RNS { namespace Utilities { + + // Thin C++ wrapper over the heatshrink C library. Configured for static + // allocation (HEATSHRINK_DYNAMIC_ALLOC=0) via platformio.ini build flags + // so no malloc happens inside heatshrink itself. The window/lookahead + // constants must match between firmware and any cooperating client (e.g. + // the JS decoder embedded in the web console). + class Compress { + + public: + // Heatshrink-encode 'in_data'. Returns the compressed bytes on + // success. Returns an empty Bytes on encoder error or when the + // compressed output would be at least as large as the input — the + // caller is expected to fall back to the uncompressed payload in + // that case, since wrapping unprofitable output only grows the wire. + static Bytes encode(const uint8_t* in_data, size_t in_len); + + // Heatshrink-decode 'in_data'. 'max_out' caps the output buffer + // growth as a defensive bound against pathological / hostile + // streams; returns an empty Bytes if decoding fails or if the + // decoded size would exceed 'max_out'. + static Bytes decode(const uint8_t* in_data, size_t in_len, size_t max_out); + + }; + +} } From 4b72dd0cc5123935a02a834f82652c0f7d9e055d Mon Sep 17 00:00:00 2001 From: Chad Attermann Date: Mon, 22 Jun 2026 09:48:29 -0600 Subject: [PATCH 3/3] Made heatshrink embedded rather than dependency --- CMakeLists.txt | 5 - library.json | 4 - platformio.ini | 1 - .../Provisioning/BuiltinNamespaces.cpp | 2 +- src/microReticulum/Utilities/Compress.cpp | 4 +- src/microReticulum/Utilities/Memory.h | 2 +- .../Utilities/heatshrink/LICENSE | 14 + .../Utilities/heatshrink/dec_sm.dot | 47 + .../Utilities/heatshrink/enc_sm.dot | 46 + .../Utilities/heatshrink/greatest.h | 837 ++++++++++++++++++ .../Utilities/heatshrink/heatshrink_common.h | 20 + .../Utilities/heatshrink/heatshrink_config.h | 36 + .../Utilities/heatshrink/heatshrink_decoder.c | 367 ++++++++ .../Utilities/heatshrink/heatshrink_decoder.h | 100 +++ .../Utilities/heatshrink/heatshrink_encoder.c | 604 +++++++++++++ .../Utilities/heatshrink/heatshrink_encoder.h | 109 +++ .../Utilities/{ => tlsf}/tlsf.c | 0 .../Utilities/{ => tlsf}/tlsf.h | 0 18 files changed, 2184 insertions(+), 14 deletions(-) create mode 100644 src/microReticulum/Utilities/heatshrink/LICENSE create mode 100644 src/microReticulum/Utilities/heatshrink/dec_sm.dot create mode 100644 src/microReticulum/Utilities/heatshrink/enc_sm.dot create mode 100644 src/microReticulum/Utilities/heatshrink/greatest.h create mode 100644 src/microReticulum/Utilities/heatshrink/heatshrink_common.h create mode 100644 src/microReticulum/Utilities/heatshrink/heatshrink_config.h create mode 100644 src/microReticulum/Utilities/heatshrink/heatshrink_decoder.c create mode 100644 src/microReticulum/Utilities/heatshrink/heatshrink_decoder.h create mode 100644 src/microReticulum/Utilities/heatshrink/heatshrink_encoder.c create mode 100644 src/microReticulum/Utilities/heatshrink/heatshrink_encoder.h rename src/microReticulum/Utilities/{ => tlsf}/tlsf.c (100%) rename src/microReticulum/Utilities/{ => tlsf}/tlsf.h (100%) diff --git a/CMakeLists.txt b/CMakeLists.txt index 48fe828..9fc30c6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -35,7 +35,6 @@ set(RNS_HEAP_POOL_BUFFER_SIZE "" CACHE STRING "TLSF heap pool size (bytes)") set(RNS_ARDUINOJSON_SOURCE_DIR "" CACHE PATH "Use local ArduinoJson checkout instead of FetchContent") set(RNS_MSGPACK_SOURCE_DIR "" CACHE PATH "Use local MsgPack checkout instead of FetchContent") set(RNS_CRYPTO_SOURCE_DIR "" CACHE PATH "Use local Crypto checkout instead of FetchContent") -set(RNS_HEATSHRINK_SOURCE_DIR "" CACHE PATH "Use local Heatshrink checkout instead of FetchContent") set(RNS_MICROSTORE_SOURCE_DIR "" CACHE PATH "Use local microStore checkout instead of FetchContent") set(RNS_ARXCONTAINER_SOURCE_DIR "" CACHE PATH "Use local ArxContainer checkout instead of FetchContent") set(RNS_ARXTYPETRAITS_SOURCE_DIR "" CACHE PATH "Use local ArxTypeTraits checkout instead of FetchContent") @@ -66,7 +65,6 @@ rns_fetch(msgpack RNS_MSGPACK_SOURCE_DIR https://github.com/hideak rns_fetch(arxcontainer RNS_ARXCONTAINER_SOURCE_DIR https://github.com/hideakitai/ArxContainer.git v0.7.0) rns_fetch(arxtypetraits RNS_ARXTYPETRAITS_SOURCE_DIR https://github.com/hideakitai/ArxTypeTraits.git v0.3.2) rns_fetch(debuglog RNS_DEBUGLOG_SOURCE_DIR https://github.com/hideakitai/DebugLog.git v0.8.4) -rns_fetch(heatshrink RNS_HEATSHRINK_SOURCE_DIR https://github.com/atomicobject/heatshrink.git v0.4.1) # Crypto and microStore have no upstream tags suitable for pinning, so pin commit SHAs. # Refresh by running `git ls-remote HEAD` and updating the hex below. rns_fetch(crypto RNS_CRYPTO_SOURCE_DIR https://github.com/attermann/Crypto.git) @@ -90,8 +88,6 @@ add_library(ArxTypeTraitsHeaders INTERFACE) target_include_directories(ArxTypeTraitsHeaders INTERFACE "${arxtypetraits_SOURCE_DIR}") add_library(DebugLogHeaders INTERFACE) target_include_directories(DebugLogHeaders INTERFACE "${debuglog_SOURCE_DIR}") -add_library(HeatshrinkHeaders INTERFACE) -target_include_directories(HeatshrinkHeaders INTERFACE "${heatshrink_SOURCE_DIR}") add_library(microStoreHeaders INTERFACE) target_include_directories(microStoreHeaders INTERFACE "${microstore_SOURCE_DIR}/include") @@ -159,7 +155,6 @@ target_link_libraries(microReticulum ArxContainerHeaders ArxTypeTraitsHeaders DebugLogHeaders - HeatshrinkHeaders microStoreHeaders CryptoLib ) diff --git a/library.json b/library.json index 382d178..fd53be1 100644 --- a/library.json +++ b/library.json @@ -27,10 +27,6 @@ "name": "MsgPack", "version": "~0.4.2" }, - { - "name": "heatshrink", - "url": "https://github.com/atomicobject/heatshrink.git#v0.4.1" - }, { "name": "Crypto", "url": "https://github.com/attermann/Crypto.git" diff --git a/platformio.ini b/platformio.ini index c98a133..26e3edf 100644 --- a/platformio.ini +++ b/platformio.ini @@ -47,7 +47,6 @@ build_flags = lib_deps = ArduinoJson@^7.4.2 MsgPack@^0.4.2 - https://github.com/atomicobject/heatshrink.git#v0.4.1 https://github.com/attermann/Crypto.git test_framework = unity ; Following property enables Unity integration testing diff --git a/src/microReticulum/Provisioning/BuiltinNamespaces.cpp b/src/microReticulum/Provisioning/BuiltinNamespaces.cpp index acccd7f..dd59df8 100644 --- a/src/microReticulum/Provisioning/BuiltinNamespaces.cpp +++ b/src/microReticulum/Provisioning/BuiltinNamespaces.cpp @@ -18,7 +18,7 @@ #include "../Reticulum.h" #include "../Transport.h" #include "../Utilities/Memory.h" -#include "../Utilities/tlsf.h" +#include "../Utilities/tlsf/tlsf.h" #include #include diff --git a/src/microReticulum/Utilities/Compress.cpp b/src/microReticulum/Utilities/Compress.cpp index f52c498..82c6f16 100644 --- a/src/microReticulum/Utilities/Compress.cpp +++ b/src/microReticulum/Utilities/Compress.cpp @@ -15,8 +15,8 @@ #include "Compress.h" extern "C" { -#include "heatshrink_encoder.h" -#include "heatshrink_decoder.h" +#include "heatshrink/heatshrink_encoder.h" +#include "heatshrink/heatshrink_decoder.h" } namespace RNS { namespace Utilities { diff --git a/src/microReticulum/Utilities/Memory.h b/src/microReticulum/Utilities/Memory.h index bc81503..b2f9f13 100644 --- a/src/microReticulum/Utilities/Memory.h +++ b/src/microReticulum/Utilities/Memory.h @@ -16,7 +16,7 @@ #include "../Log.h" -#include "tlsf.h" +#include "tlsf/tlsf.h" #include diff --git a/src/microReticulum/Utilities/heatshrink/LICENSE b/src/microReticulum/Utilities/heatshrink/LICENSE new file mode 100644 index 0000000..6b69634 --- /dev/null +++ b/src/microReticulum/Utilities/heatshrink/LICENSE @@ -0,0 +1,14 @@ +Copyright (c) 2013-2015, Scott Vokes +All rights reserved. + +Permission to use, copy, modify, and/or distribute this software for any +purpose with or without fee is hereby granted, provided that the above +copyright notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. diff --git a/src/microReticulum/Utilities/heatshrink/dec_sm.dot b/src/microReticulum/Utilities/heatshrink/dec_sm.dot new file mode 100644 index 0000000..21e56b9 --- /dev/null +++ b/src/microReticulum/Utilities/heatshrink/dec_sm.dot @@ -0,0 +1,47 @@ +digraph { + graph [label="Decoder state machine", labelloc="t"] + Start [style="invis", shape="point"] + tag_bit + yield_literal + backref_index_msb + backref_index_lsb + backref_count_msb + backref_count_lsb + yield_backref + done [peripheries=2] + + tag_bit->tag_bit [label="sink()", color="blue", weight=10] + Start->tag_bit + + tag_bit->yield_literal [label="pop 1-bit"] + tag_bit->backref_index_msb [label="pop 0-bit", weight=10] + tag_bit->backref_index_lsb [label="pop 0-bit, index <8 bits", weight=10] + + yield_literal->yield_literal [label="sink()", color="blue"] + yield_literal->yield_literal [label="poll()", color="red"] + yield_literal->tag_bit [label="poll(), done", color="red"] + + backref_index_msb->backref_index_msb [label="sink()", color="blue"] + backref_index_msb->backref_index_lsb [label="pop index, upper bits", weight=10] + backref_index_msb->done [label="finish()", color="blue"] + + backref_index_lsb->backref_index_lsb [label="sink()", color="blue"] + backref_index_lsb->backref_count_msb [label="pop index, lower bits", weight=10] + backref_index_lsb->backref_count_lsb [label="pop index, count <=8 bits", weight=10] + backref_index_lsb->done [label="finish()", color="blue"] + + backref_count_msb->backref_count_msb [label="sink()", color="blue"] + backref_count_msb->backref_count_lsb [label="pop count, upper bits", weight=10] + backref_count_msb->done [label="finish()", color="blue"] + + backref_count_lsb->backref_count_lsb [label="sink()", color="blue"] + backref_count_lsb->yield_backref [label="pop count, lower bits", weight=10] + backref_count_lsb->done [label="finish()", color="blue"] + + yield_backref->yield_backref [label="sink()", color="blue"] + yield_backref->yield_backref [label="poll()", color="red"] + yield_backref->tag_bit [label="poll(), done", + color="red", weight=10] + + tag_bit->done [label="finish()", color="blue"] +} diff --git a/src/microReticulum/Utilities/heatshrink/enc_sm.dot b/src/microReticulum/Utilities/heatshrink/enc_sm.dot new file mode 100644 index 0000000..748ec47 --- /dev/null +++ b/src/microReticulum/Utilities/heatshrink/enc_sm.dot @@ -0,0 +1,46 @@ +digraph { + graph [label="Encoder state machine", labelloc="t"] + start [style="invis", shape="point"] + not_full + filled + search + yield_tag_bit + yield_literal + yield_br_length + yield_br_index + save_backlog + flush_bits + done [peripheries=2] + + start->not_full [label="start"] + + not_full->not_full [label="sink(), not full", color="blue"] + not_full->filled [label="sink(), buffer is full", color="blue"] + not_full->filled [label="finish(), set is_finished", color="blue"] + + filled->search [label="indexing (if any)"] + + search->yield_tag_bit [label="literal"] + search->yield_tag_bit [label="match found"] + search->save_backlog [label="input exhausted, not finishing"] + search->flush_bits [label="input exhausted, finishing"] + + yield_tag_bit->yield_tag_bit [label="poll(), full buf", color="red"] + yield_tag_bit->yield_literal [label="poll(), literal", color="red"] + yield_tag_bit->yield_br_index [label="poll(), match", color="red"] + + yield_literal->yield_literal [label="poll(), full buf", color="red"] + yield_literal->search [label="done"] + + yield_br_index->yield_br_index [label="poll(), full buf", color="red"] + yield_br_index->yield_br_length [label="poll()", color="red"] + + yield_br_length->yield_br_length [label="poll(), full buf", color="red"] + yield_br_length->search [label="done"] + + save_backlog->not_full [label="expect more input"] + + flush_bits->flush_bits [label="poll(), full buf", color="red"] + flush_bits->done [label="poll(), flushed", color="red"] + flush_bits->done [label="no more output"] +} diff --git a/src/microReticulum/Utilities/heatshrink/greatest.h b/src/microReticulum/Utilities/heatshrink/greatest.h new file mode 100644 index 0000000..b3c4d9e --- /dev/null +++ b/src/microReticulum/Utilities/heatshrink/greatest.h @@ -0,0 +1,837 @@ +/* + * Copyright (c) 2011-2015 Scott Vokes + * + * Permission to use, copy, modify, and/or distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ + +#ifndef GREATEST_H +#define GREATEST_H + +/* 1.0.0 */ +#define GREATEST_VERSION_MAJOR 1 +#define GREATEST_VERSION_MINOR 0 +#define GREATEST_VERSION_PATCH 0 + +/* A unit testing system for C, contained in 1 file. + * It doesn't use dynamic allocation or depend on anything + * beyond ANSI C89. */ + + +/********************************************************************* + * Minimal test runner template + *********************************************************************/ +#if 0 + +#include "greatest.h" + +TEST foo_should_foo() { + PASS(); +} + +static void setup_cb(void *data) { + printf("setup callback for each test case\n"); +} + +static void teardown_cb(void *data) { + printf("teardown callback for each test case\n"); +} + +SUITE(suite) { + /* Optional setup/teardown callbacks which will be run before/after + * every test case in the suite. + * Cleared when the suite finishes. */ + SET_SETUP(setup_cb, voidp_to_callback_data); + SET_TEARDOWN(teardown_cb, voidp_to_callback_data); + + RUN_TEST(foo_should_foo); +} + +/* Add definitions that need to be in the test runner's main file. */ +GREATEST_MAIN_DEFS(); + +/* Set up, run suite(s) of tests, report pass/fail/skip stats. */ +int run_tests(void) { + GREATEST_INIT(); /* init. greatest internals */ + /* List of suites to run. */ + RUN_SUITE(suite); + GREATEST_REPORT(); /* display results */ + return greatest_all_passed(); +} + +/* main(), for a standalone command-line test runner. + * This replaces run_tests above, and adds command line option + * handling and exiting with a pass/fail status. */ +int main(int argc, char **argv) { + GREATEST_MAIN_BEGIN(); /* init & parse command-line args */ + RUN_SUITE(suite); + GREATEST_MAIN_END(); /* display results */ +} + +#endif +/*********************************************************************/ + + +#include +#include +#include + +/*********** + * Options * + ***********/ + +/* Default column width for non-verbose output. */ +#ifndef GREATEST_DEFAULT_WIDTH +#define GREATEST_DEFAULT_WIDTH 72 +#endif + +/* FILE *, for test logging. */ +#ifndef GREATEST_STDOUT +#define GREATEST_STDOUT stdout +#endif + +/* Remove GREATEST_ prefix from most commonly used symbols? */ +#ifndef GREATEST_USE_ABBREVS +#define GREATEST_USE_ABBREVS 1 +#endif + +/* Set to 0 to disable all use of setjmp/longjmp. */ +#ifndef GREATEST_USE_LONGJMP +#define GREATEST_USE_LONGJMP 1 +#endif + +#if GREATEST_USE_LONGJMP +#include +#endif + +/* Set to 0 to disable all use of time.h / clock(). */ +#ifndef GREATEST_USE_TIME +#define GREATEST_USE_TIME 1 +#endif + +#if GREATEST_USE_TIME +#include +#endif + +/* Floating point type, for ASSERT_IN_RANGE. */ +#ifndef GREATEST_FLOAT +#define GREATEST_FLOAT double +#define GREATEST_FLOAT_FMT "%g" +#endif + +/********* + * Types * + *********/ + +/* Info for the current running suite. */ +typedef struct greatest_suite_info { + unsigned int tests_run; + unsigned int passed; + unsigned int failed; + unsigned int skipped; + +#if GREATEST_USE_TIME + /* timers, pre/post running suite and individual tests */ + clock_t pre_suite; + clock_t post_suite; + clock_t pre_test; + clock_t post_test; +#endif +} greatest_suite_info; + +/* Type for a suite function. */ +typedef void (greatest_suite_cb)(void); + +/* Types for setup/teardown callbacks. If non-NULL, these will be run + * and passed the pointer to their additional data. */ +typedef void (greatest_setup_cb)(void *udata); +typedef void (greatest_teardown_cb)(void *udata); + +/* Type for an equality comparison between two pointers of the same type. + * Should return non-0 if equal, otherwise 0. + * UDATA is a closure value, passed through from ASSERT_EQUAL_T[m]. */ +typedef int greatest_equal_cb(const void *exp, const void *got, void *udata); + +/* Type for a callback that prints a value pointed to by T. + * Return value has the same meaning as printf's. + * UDATA is a closure value, passed through from ASSERT_EQUAL_T[m]. */ +typedef int greatest_printf_cb(const void *t, void *udata); + +/* Callbacks for an arbitrary type; needed for type-specific + * comparisons via GREATEST_ASSERT_EQUAL_T[m].*/ +typedef struct greatest_type_info { + greatest_equal_cb *equal; + greatest_printf_cb *print; +} greatest_type_info; + +/* Callbacks for string type. */ +extern greatest_type_info greatest_type_info_string; + +typedef enum { + GREATEST_FLAG_VERBOSE = 0x01, + GREATEST_FLAG_FIRST_FAIL = 0x02, + GREATEST_FLAG_LIST_ONLY = 0x04 +} GREATEST_FLAG; + +/* Struct containing all test runner state. */ +typedef struct greatest_run_info { + unsigned int flags; + unsigned int tests_run; /* total test count */ + + /* overall pass/fail/skip counts */ + unsigned int passed; + unsigned int failed; + unsigned int skipped; + unsigned int assertions; + + /* currently running test suite */ + greatest_suite_info suite; + + /* info to print about the most recent failure */ + const char *fail_file; + unsigned int fail_line; + const char *msg; + + /* current setup/teardown hooks and userdata */ + greatest_setup_cb *setup; + void *setup_udata; + greatest_teardown_cb *teardown; + void *teardown_udata; + + /* formatting info for ".....s...F"-style output */ + unsigned int col; + unsigned int width; + + /* only run a specific suite or test */ + char *suite_filter; + char *test_filter; + +#if GREATEST_USE_TIME + /* overall timers */ + clock_t begin; + clock_t end; +#endif + +#if GREATEST_USE_LONGJMP + jmp_buf jump_dest; +#endif +} greatest_run_info; + +/* Global var for the current testing context. + * Initialized by GREATEST_MAIN_DEFS(). */ +extern greatest_run_info greatest_info; + + +/********************** + * Exported functions * + **********************/ + +/* These are used internally by greatest. */ +void greatest_do_pass(const char *name); +void greatest_do_fail(const char *name); +void greatest_do_skip(const char *name); +int greatest_pre_test(const char *name); +void greatest_post_test(const char *name, int res); +void greatest_usage(const char *name); +int greatest_do_assert_equal_t(const void *exp, const void *got, + greatest_type_info *type_info, void *udata); + +/* These are part of the public greatest API. */ +void GREATEST_SET_SETUP_CB(greatest_setup_cb *cb, void *udata); +void GREATEST_SET_TEARDOWN_CB(greatest_teardown_cb *cb, void *udata); +int greatest_all_passed(void); + + +/******************** +* Language Support * +********************/ + +/* If __VA_ARGS__ (C99) is supported, allow parametric testing +* without needing to manually manage the argument struct. */ +#if __STDC_VERSION__ >= 19901L || _MSC_VER >= 1800 +#define GREATEST_VA_ARGS +#endif + + +/********** + * Macros * + **********/ + +/* Define a suite. */ +#define GREATEST_SUITE(NAME) void NAME(void); void NAME(void) + +/* Start defining a test function. + * The arguments are not included, to allow parametric testing. */ +#define GREATEST_TEST static greatest_test_res + +/* PASS/FAIL/SKIP result from a test. Used internally. */ +typedef enum { + GREATEST_TEST_RES_PASS = 0, + GREATEST_TEST_RES_FAIL = -1, + GREATEST_TEST_RES_SKIP = 1 +} greatest_test_res; + +/* Run a suite. */ +#define GREATEST_RUN_SUITE(S_NAME) greatest_run_suite(S_NAME, #S_NAME) + +/* Run a test in the current suite. */ +#define GREATEST_RUN_TEST(TEST) \ + do { \ + if (greatest_pre_test(#TEST) == 1) { \ + greatest_test_res res = GREATEST_SAVE_CONTEXT(); \ + if (res == GREATEST_TEST_RES_PASS) { \ + res = TEST(); \ + } \ + greatest_post_test(#TEST, res); \ + } else if (GREATEST_LIST_ONLY()) { \ + fprintf(GREATEST_STDOUT, " %s\n", #TEST); \ + } \ + } while (0) + +/* Run a test in the current suite with one void * argument, + * which can be a pointer to a struct with multiple arguments. */ +#define GREATEST_RUN_TEST1(TEST, ENV) \ + do { \ + if (greatest_pre_test(#TEST) == 1) { \ + int res = TEST(ENV); \ + greatest_post_test(#TEST, res); \ + } else if (GREATEST_LIST_ONLY()) { \ + fprintf(GREATEST_STDOUT, " %s\n", #TEST); \ + } \ + } while (0) + +#ifdef GREATEST_VA_ARGS +#define GREATEST_RUN_TESTp(TEST, ...) \ + do { \ + if (greatest_pre_test(#TEST) == 1) { \ + int res = TEST(__VA_ARGS__); \ + greatest_post_test(#TEST, res); \ + } else if (GREATEST_LIST_ONLY()) { \ + fprintf(GREATEST_STDOUT, " %s\n", #TEST); \ + } \ + } while (0) +#endif + + +/* Check if the test runner is in verbose mode. */ +#define GREATEST_IS_VERBOSE() (greatest_info.flags & GREATEST_FLAG_VERBOSE) +#define GREATEST_LIST_ONLY() (greatest_info.flags & GREATEST_FLAG_LIST_ONLY) +#define GREATEST_FIRST_FAIL() (greatest_info.flags & GREATEST_FLAG_FIRST_FAIL) +#define GREATEST_FAILURE_ABORT() (greatest_info.suite.failed > 0 && GREATEST_FIRST_FAIL()) + +/* Message-less forms of tests defined below. */ +#define GREATEST_PASS() GREATEST_PASSm(NULL) +#define GREATEST_FAIL() GREATEST_FAILm(NULL) +#define GREATEST_SKIP() GREATEST_SKIPm(NULL) +#define GREATEST_ASSERT(COND) \ + GREATEST_ASSERTm(#COND, COND) +#define GREATEST_ASSERT_OR_LONGJMP(COND) \ + GREATEST_ASSERT_OR_LONGJMPm(#COND, COND) +#define GREATEST_ASSERT_FALSE(COND) \ + GREATEST_ASSERT_FALSEm(#COND, COND) +#define GREATEST_ASSERT_EQ(EXP, GOT) \ + GREATEST_ASSERT_EQm(#EXP " != " #GOT, EXP, GOT) +#define GREATEST_ASSERT_EQ_FMT(EXP, GOT, FMT) \ + GREATEST_ASSERT_EQ_FMTm(#EXP " != " #GOT, EXP, GOT, FMT) +#define GREATEST_ASSERT_IN_RANGE(EXP, GOT, TOL) \ + GREATEST_ASSERT_IN_RANGEm(#EXP " != " #GOT " +/- " #TOL, EXP, GOT, TOL) +#define GREATEST_ASSERT_EQUAL_T(EXP, GOT, TYPE_INFO, UDATA) \ + GREATEST_ASSERT_EQUAL_Tm(#EXP " != " #GOT, EXP, GOT, TYPE_INFO, UDATA) +#define GREATEST_ASSERT_STR_EQ(EXP, GOT) \ + GREATEST_ASSERT_STR_EQm(#EXP " != " #GOT, EXP, GOT) + +/* The following forms take an additional message argument first, + * to be displayed by the test runner. */ + +/* Fail if a condition is not true, with message. */ +#define GREATEST_ASSERTm(MSG, COND) \ + do { \ + greatest_info.assertions++; \ + if (!(COND)) { GREATEST_FAILm(MSG); } \ + } while (0) + +/* Fail if a condition is not true, longjmping out of test. */ +#define GREATEST_ASSERT_OR_LONGJMPm(MSG, COND) \ + do { \ + greatest_info.assertions++; \ + if (!(COND)) { GREATEST_FAIL_WITH_LONGJMPm(MSG); } \ + } while (0) + +/* Fail if a condition is not false, with message. */ +#define GREATEST_ASSERT_FALSEm(MSG, COND) \ + do { \ + greatest_info.assertions++; \ + if ((COND)) { GREATEST_FAILm(MSG); } \ + } while (0) + +/* Fail if EXP != GOT (equality comparison by ==). */ +#define GREATEST_ASSERT_EQm(MSG, EXP, GOT) \ + do { \ + greatest_info.assertions++; \ + if ((EXP) != (GOT)) { GREATEST_FAILm(MSG); } \ + } while (0) + +/* Fail if EXP != GOT (equality comparison by ==). */ +#define GREATEST_ASSERT_EQ_FMTm(MSG, EXP, GOT, FMT) \ + do { \ + greatest_info.assertions++; \ + const char *fmt = ( FMT ); \ + if ((EXP) != (GOT)) { \ + fprintf(GREATEST_STDOUT, "\nExpected: "); \ + fprintf(GREATEST_STDOUT, fmt, EXP); \ + fprintf(GREATEST_STDOUT, "\nGot: "); \ + fprintf(GREATEST_STDOUT, fmt, GOT); \ + fprintf(GREATEST_STDOUT, "\n"); \ + GREATEST_FAILm(MSG); \ + } \ + } while (0) + +/* Fail if GOT not in range of EXP +|- TOL. */ +#define GREATEST_ASSERT_IN_RANGEm(MSG, EXP, GOT, TOL) \ + do { \ + greatest_info.assertions++; \ + GREATEST_FLOAT exp = (EXP); \ + GREATEST_FLOAT got = (GOT); \ + GREATEST_FLOAT tol = (TOL); \ + if ((exp > got && exp - got > tol) || \ + (exp < got && got - exp > tol)) { \ + fprintf(GREATEST_STDOUT, \ + "\nExpected: " GREATEST_FLOAT_FMT \ + " +/- " GREATEST_FLOAT_FMT "\n" \ + "Got: " GREATEST_FLOAT_FMT "\n", \ + exp, tol, got); \ + GREATEST_FAILm(MSG); \ + } \ + } while (0) + +/* Fail if EXP is not equal to GOT, according to strcmp. */ +#define GREATEST_ASSERT_STR_EQm(MSG, EXP, GOT) \ + do { \ + GREATEST_ASSERT_EQUAL_Tm(MSG, EXP, GOT, \ + &greatest_type_info_string, NULL); \ + } while (0) \ + +/* Fail if EXP is not equal to GOT, according to a comparison + * callback in TYPE_INFO. If they are not equal, optionally use a + * print callback in TYPE_INFO to print them. */ +#define GREATEST_ASSERT_EQUAL_Tm(MSG, EXP, GOT, TYPE_INFO, UDATA) \ + do { \ + greatest_type_info *type_info = (TYPE_INFO); \ + greatest_info.assertions++; \ + if (!greatest_do_assert_equal_t(EXP, GOT, \ + type_info, UDATA)) { \ + if (type_info == NULL || type_info->equal == NULL) { \ + GREATEST_FAILm("type_info->equal callback missing!"); \ + } else { \ + GREATEST_FAILm(MSG); \ + } \ + } \ + } while (0) \ + +/* Pass. */ +#define GREATEST_PASSm(MSG) \ + do { \ + greatest_info.msg = MSG; \ + return GREATEST_TEST_RES_PASS; \ + } while (0) + +/* Fail. */ +#define GREATEST_FAILm(MSG) \ + do { \ + greatest_info.fail_file = __FILE__; \ + greatest_info.fail_line = __LINE__; \ + greatest_info.msg = MSG; \ + return GREATEST_TEST_RES_FAIL; \ + } while (0) + +/* Optional GREATEST_FAILm variant that longjmps. */ +#if GREATEST_USE_LONGJMP +#define GREATEST_FAIL_WITH_LONGJMP() GREATEST_FAIL_WITH_LONGJMPm(NULL) +#define GREATEST_FAIL_WITH_LONGJMPm(MSG) \ + do { \ + greatest_info.fail_file = __FILE__; \ + greatest_info.fail_line = __LINE__; \ + greatest_info.msg = MSG; \ + longjmp(greatest_info.jump_dest, GREATEST_TEST_RES_FAIL); \ + } while (0) +#endif + +/* Skip the current test. */ +#define GREATEST_SKIPm(MSG) \ + do { \ + greatest_info.msg = MSG; \ + return GREATEST_TEST_RES_SKIP; \ + } while (0) + +/* Check the result of a subfunction using ASSERT, etc. */ +#define GREATEST_CHECK_CALL(RES) \ + do { \ + int _check_call_res = RES; \ + if (_check_call_res != GREATEST_TEST_RES_PASS) { \ + return _check_call_res; \ + } \ + } while (0) \ + +#if GREATEST_USE_TIME +#define GREATEST_SET_TIME(NAME) \ + NAME = clock(); \ + if (NAME == (clock_t) -1) { \ + fprintf(GREATEST_STDOUT, \ + "clock error: %s\n", #NAME); \ + exit(EXIT_FAILURE); \ + } + +#define GREATEST_CLOCK_DIFF(C1, C2) \ + fprintf(GREATEST_STDOUT, " (%lu ticks, %.3f sec)", \ + (long unsigned int) (C2) - (long unsigned int)(C1), \ + (double)((C2) - (C1)) / (1.0 * (double)CLOCKS_PER_SEC)) +#else +#define GREATEST_SET_TIME(UNUSED) +#define GREATEST_CLOCK_DIFF(UNUSED1, UNUSED2) +#endif + +#if GREATEST_USE_LONGJMP +#define GREATEST_SAVE_CONTEXT() \ + /* setjmp returns 0 (GREATEST_TEST_RES_PASS) on first call */ \ + /* so the test runs, then RES_FAIL from FAIL_WITH_LONGJMP. */ \ + ((greatest_test_res)(setjmp(greatest_info.jump_dest))) +#else +#define GREATEST_SAVE_CONTEXT() \ + /*a no-op, since setjmp/longjmp aren't being used */ \ + GREATEST_TEST_RES_PASS +#endif + +/* Include several function definitions in the main test file. */ +#define GREATEST_MAIN_DEFS() \ + \ +/* Is FILTER a subset of NAME? */ \ +static int greatest_name_match(const char *name, \ + const char *filter) { \ + size_t offset = 0; \ + size_t filter_len = strlen(filter); \ + while (name[offset] != '\0') { \ + if (name[offset] == filter[0]) { \ + if (0 == strncmp(&name[offset], filter, filter_len)) { \ + return 1; \ + } \ + } \ + offset++; \ + } \ + \ + return 0; \ +} \ + \ +int greatest_pre_test(const char *name) { \ + if (!GREATEST_LIST_ONLY() \ + && (!GREATEST_FIRST_FAIL() || greatest_info.suite.failed == 0) \ + && (greatest_info.test_filter == NULL || \ + greatest_name_match(name, greatest_info.test_filter))) { \ + GREATEST_SET_TIME(greatest_info.suite.pre_test); \ + if (greatest_info.setup) { \ + greatest_info.setup(greatest_info.setup_udata); \ + } \ + return 1; /* test should be run */ \ + } else { \ + return 0; /* skipped */ \ + } \ +} \ + \ +void greatest_post_test(const char *name, int res) { \ + GREATEST_SET_TIME(greatest_info.suite.post_test); \ + if (greatest_info.teardown) { \ + void *udata = greatest_info.teardown_udata; \ + greatest_info.teardown(udata); \ + } \ + \ + if (res <= GREATEST_TEST_RES_FAIL) { \ + greatest_do_fail(name); \ + } else if (res >= GREATEST_TEST_RES_SKIP) { \ + greatest_do_skip(name); \ + } else if (res == GREATEST_TEST_RES_PASS) { \ + greatest_do_pass(name); \ + } \ + greatest_info.suite.tests_run++; \ + greatest_info.col++; \ + if (GREATEST_IS_VERBOSE()) { \ + GREATEST_CLOCK_DIFF(greatest_info.suite.pre_test, \ + greatest_info.suite.post_test); \ + fprintf(GREATEST_STDOUT, "\n"); \ + } else if (greatest_info.col % greatest_info.width == 0) { \ + fprintf(GREATEST_STDOUT, "\n"); \ + greatest_info.col = 0; \ + } \ + if (GREATEST_STDOUT == stdout) fflush(stdout); \ +} \ + \ +static void greatest_run_suite(greatest_suite_cb *suite_cb, \ + const char *suite_name) { \ + if (greatest_info.suite_filter && \ + !greatest_name_match(suite_name, greatest_info.suite_filter)) { \ + return; \ + } \ + if (GREATEST_FIRST_FAIL() && greatest_info.failed > 0) { return; } \ + memset(&greatest_info.suite, 0, sizeof(greatest_info.suite)); \ + greatest_info.col = 0; \ + fprintf(GREATEST_STDOUT, "\n* Suite %s:\n", suite_name); \ + GREATEST_SET_TIME(greatest_info.suite.pre_suite); \ + suite_cb(); \ + GREATEST_SET_TIME(greatest_info.suite.post_suite); \ + if (greatest_info.suite.tests_run > 0) { \ + fprintf(GREATEST_STDOUT, \ + "\n%u tests - %u pass, %u fail, %u skipped", \ + greatest_info.suite.tests_run, \ + greatest_info.suite.passed, \ + greatest_info.suite.failed, \ + greatest_info.suite.skipped); \ + GREATEST_CLOCK_DIFF(greatest_info.suite.pre_suite, \ + greatest_info.suite.post_suite); \ + fprintf(GREATEST_STDOUT, "\n"); \ + } \ + greatest_info.setup = NULL; \ + greatest_info.setup_udata = NULL; \ + greatest_info.teardown = NULL; \ + greatest_info.teardown_udata = NULL; \ + greatest_info.passed += greatest_info.suite.passed; \ + greatest_info.failed += greatest_info.suite.failed; \ + greatest_info.skipped += greatest_info.suite.skipped; \ + greatest_info.tests_run += greatest_info.suite.tests_run; \ +} \ + \ +void greatest_do_pass(const char *name) { \ + if (GREATEST_IS_VERBOSE()) { \ + fprintf(GREATEST_STDOUT, "PASS %s: %s", \ + name, greatest_info.msg ? greatest_info.msg : ""); \ + } else { \ + fprintf(GREATEST_STDOUT, "."); \ + } \ + greatest_info.suite.passed++; \ +} \ + \ +void greatest_do_fail(const char *name) { \ + if (GREATEST_IS_VERBOSE()) { \ + fprintf(GREATEST_STDOUT, \ + "FAIL %s: %s (%s:%u)", \ + name, greatest_info.msg ? greatest_info.msg : "", \ + greatest_info.fail_file, greatest_info.fail_line); \ + } else { \ + fprintf(GREATEST_STDOUT, "F"); \ + greatest_info.col++; \ + /* add linebreak if in line of '.'s */ \ + if (greatest_info.col != 0) { \ + fprintf(GREATEST_STDOUT, "\n"); \ + greatest_info.col = 0; \ + } \ + fprintf(GREATEST_STDOUT, "FAIL %s: %s (%s:%u)\n", \ + name, \ + greatest_info.msg ? greatest_info.msg : "", \ + greatest_info.fail_file, greatest_info.fail_line); \ + } \ + greatest_info.suite.failed++; \ +} \ + \ +void greatest_do_skip(const char *name) { \ + if (GREATEST_IS_VERBOSE()) { \ + fprintf(GREATEST_STDOUT, "SKIP %s: %s", \ + name, \ + greatest_info.msg ? \ + greatest_info.msg : "" ); \ + } else { \ + fprintf(GREATEST_STDOUT, "s"); \ + } \ + greatest_info.suite.skipped++; \ +} \ + \ +int greatest_do_assert_equal_t(const void *exp, const void *got, \ + greatest_type_info *type_info, void *udata) { \ + int eq = 0; \ + if (type_info == NULL || type_info->equal == NULL) { \ + return 0; \ + } \ + eq = type_info->equal(exp, got, udata); \ + if (!eq) { \ + if (type_info->print != NULL) { \ + fprintf(GREATEST_STDOUT, "\nExpected: "); \ + (void)type_info->print(exp, udata); \ + fprintf(GREATEST_STDOUT, "\nGot: "); \ + (void)type_info->print(got, udata); \ + fprintf(GREATEST_STDOUT, "\n"); \ + } else { \ + fprintf(GREATEST_STDOUT, \ + "GREATEST_ASSERT_EQUAL_T failure at %s:%dn", \ + greatest_info.fail_file, \ + greatest_info.fail_line); \ + } \ + } \ + return eq; \ +} \ + \ +void greatest_usage(const char *name) { \ + fprintf(GREATEST_STDOUT, \ + "Usage: %s [-hlfv] [-s SUITE] [-t TEST]\n" \ + " -h print this Help\n" \ + " -l List suites and their tests, then exit\n" \ + " -f Stop runner after first failure\n" \ + " -v Verbose output\n" \ + " -s SUITE only run suite named SUITE\n" \ + " -t TEST only run test named TEST\n", \ + name); \ +} \ + \ +int greatest_all_passed() { return (greatest_info.failed == 0); } \ + \ +void GREATEST_SET_SETUP_CB(greatest_setup_cb *cb, void *udata) { \ + greatest_info.setup = cb; \ + greatest_info.setup_udata = udata; \ +} \ + \ +void GREATEST_SET_TEARDOWN_CB(greatest_teardown_cb *cb, \ + void *udata) { \ + greatest_info.teardown = cb; \ + greatest_info.teardown_udata = udata; \ +} \ + \ +static int greatest_string_equal_cb(const void *exp, const void *got, \ + void *udata) { \ + (void)udata; \ + return (0 == strcmp((const char *)exp, (const char *)got)); \ +} \ + \ +static int greatest_string_printf_cb(const void *t, void *udata) { \ + (void)udata; \ + return fprintf(GREATEST_STDOUT, "%s", (const char *)t); \ +} \ + \ +greatest_type_info greatest_type_info_string = { \ + greatest_string_equal_cb, \ + greatest_string_printf_cb, \ +}; \ + \ +greatest_run_info greatest_info + +/* Init internals. */ +#define GREATEST_INIT() \ + do { \ + memset(&greatest_info, 0, sizeof(greatest_info)); \ + greatest_info.width = GREATEST_DEFAULT_WIDTH; \ + GREATEST_SET_TIME(greatest_info.begin); \ + } while (0) \ + +/* Handle command-line arguments, etc. */ +#define GREATEST_MAIN_BEGIN() \ + do { \ + int i = 0; \ + GREATEST_INIT(); \ + for (i = 1; i < argc; i++) { \ + if (0 == strcmp("-t", argv[i])) { \ + if (argc <= i + 1) { \ + greatest_usage(argv[0]); \ + exit(EXIT_FAILURE); \ + } \ + greatest_info.test_filter = argv[i+1]; \ + i++; \ + } else if (0 == strcmp("-s", argv[i])) { \ + if (argc <= i + 1) { \ + greatest_usage(argv[0]); \ + exit(EXIT_FAILURE); \ + } \ + greatest_info.suite_filter = argv[i+1]; \ + i++; \ + } else if (0 == strcmp("-f", argv[i])) { \ + greatest_info.flags |= GREATEST_FLAG_FIRST_FAIL; \ + } else if (0 == strcmp("-v", argv[i])) { \ + greatest_info.flags |= GREATEST_FLAG_VERBOSE; \ + } else if (0 == strcmp("-l", argv[i])) { \ + greatest_info.flags |= GREATEST_FLAG_LIST_ONLY; \ + } else if (0 == strcmp("-h", argv[i])) { \ + greatest_usage(argv[0]); \ + exit(EXIT_SUCCESS); \ + } else { \ + fprintf(GREATEST_STDOUT, \ + "Unknown argument '%s'\n", argv[i]); \ + greatest_usage(argv[0]); \ + exit(EXIT_FAILURE); \ + } \ + } \ + } while (0) + +/* Report passes, failures, skipped tests, the number of + * assertions, and the overall run time. */ +#define GREATEST_REPORT() \ + do { \ + if (!GREATEST_LIST_ONLY()) { \ + GREATEST_SET_TIME(greatest_info.end); \ + fprintf(GREATEST_STDOUT, \ + "\nTotal: %u tests", greatest_info.tests_run); \ + GREATEST_CLOCK_DIFF(greatest_info.begin, \ + greatest_info.end); \ + fprintf(GREATEST_STDOUT, ", %u assertions\n", \ + greatest_info.assertions); \ + fprintf(GREATEST_STDOUT, \ + "Pass: %u, fail: %u, skip: %u.\n", \ + greatest_info.passed, \ + greatest_info.failed, greatest_info.skipped); \ + } \ + } while (0) + +/* Report results, exit with exit status based on results. */ +#define GREATEST_MAIN_END() \ + do { \ + GREATEST_REPORT(); \ + return (greatest_all_passed() ? EXIT_SUCCESS : EXIT_FAILURE); \ + } while (0) + +/* Make abbreviations without the GREATEST_ prefix for the + * most commonly used symbols. */ +#if GREATEST_USE_ABBREVS +#define TEST GREATEST_TEST +#define SUITE GREATEST_SUITE +#define RUN_TEST GREATEST_RUN_TEST +#define RUN_TEST1 GREATEST_RUN_TEST1 +#define RUN_SUITE GREATEST_RUN_SUITE +#define ASSERT GREATEST_ASSERT +#define ASSERTm GREATEST_ASSERTm +#define ASSERT_FALSE GREATEST_ASSERT_FALSE +#define ASSERT_EQ GREATEST_ASSERT_EQ +#define ASSERT_EQ_FMT GREATEST_ASSERT_EQ_FMT +#define ASSERT_IN_RANGE GREATEST_ASSERT_IN_RANGE +#define ASSERT_EQUAL_T GREATEST_ASSERT_EQUAL_T +#define ASSERT_STR_EQ GREATEST_ASSERT_STR_EQ +#define ASSERT_FALSEm GREATEST_ASSERT_FALSEm +#define ASSERT_EQm GREATEST_ASSERT_EQm +#define ASSERT_EQ_FMTm GREATEST_ASSERT_EQ_FMTm +#define ASSERT_IN_RANGEm GREATEST_ASSERT_IN_RANGEm +#define ASSERT_EQUAL_Tm GREATEST_ASSERT_EQUAL_Tm +#define ASSERT_STR_EQm GREATEST_ASSERT_STR_EQm +#define PASS GREATEST_PASS +#define FAIL GREATEST_FAIL +#define SKIP GREATEST_SKIP +#define PASSm GREATEST_PASSm +#define FAILm GREATEST_FAILm +#define SKIPm GREATEST_SKIPm +#define SET_SETUP GREATEST_SET_SETUP_CB +#define SET_TEARDOWN GREATEST_SET_TEARDOWN_CB +#define CHECK_CALL GREATEST_CHECK_CALL + +#ifdef GREATEST_VA_ARGS +#define RUN_TESTp GREATEST_RUN_TESTp +#endif + +#if GREATEST_USE_LONGJMP +#define ASSERT_OR_LONGJMP GREATEST_ASSERT_OR_LONGJMP +#define ASSERT_OR_LONGJMPm GREATEST_ASSERT_OR_LONGJMPm +#define FAIL_WITH_LONGJMP GREATEST_FAIL_WITH_LONGJMP +#define FAIL_WITH_LONGJMPm GREATEST_FAIL_WITH_LONGJMPm +#endif + +#endif /* USE_ABBREVS */ + +#endif diff --git a/src/microReticulum/Utilities/heatshrink/heatshrink_common.h b/src/microReticulum/Utilities/heatshrink/heatshrink_common.h new file mode 100644 index 0000000..243f447 --- /dev/null +++ b/src/microReticulum/Utilities/heatshrink/heatshrink_common.h @@ -0,0 +1,20 @@ +#ifndef HEATSHRINK_H +#define HEATSHRINK_H + +#define HEATSHRINK_AUTHOR "Scott Vokes " +#define HEATSHRINK_URL "https://github.com/atomicobject/heatshrink" + +/* Version 0.4.1 */ +#define HEATSHRINK_VERSION_MAJOR 0 +#define HEATSHRINK_VERSION_MINOR 4 +#define HEATSHRINK_VERSION_PATCH 1 + +#define HEATSHRINK_MIN_WINDOW_BITS 4 +#define HEATSHRINK_MAX_WINDOW_BITS 15 + +#define HEATSHRINK_MIN_LOOKAHEAD_BITS 3 + +#define HEATSHRINK_LITERAL_MARKER 0x01 +#define HEATSHRINK_BACKREF_MARKER 0x00 + +#endif diff --git a/src/microReticulum/Utilities/heatshrink/heatshrink_config.h b/src/microReticulum/Utilities/heatshrink/heatshrink_config.h new file mode 100644 index 0000000..fa60109 --- /dev/null +++ b/src/microReticulum/Utilities/heatshrink/heatshrink_config.h @@ -0,0 +1,36 @@ +#ifndef HEATSHRINK_CONFIG_H +#define HEATSHRINK_CONFIG_H + +/* +microReticulum Changes: +Avoids malloc, fixes window/lookahead at compile time. +Window=9 (512-byte history) + lookahead=4 trades modest RAM (~3 KB per encoder, ~1 KB per decoder) for a solid compression ratio on text-like MsgPack payloads. +NOTE: Keep these in sync with whatever Utilities/Compress allocates and with the JS decoder's window assumption in webconsole. +*/ + +/* Should functionality assuming dynamic allocation be used? */ +#ifndef HEATSHRINK_DYNAMIC_ALLOC +//#define HEATSHRINK_DYNAMIC_ALLOC 1 +#define HEATSHRINK_DYNAMIC_ALLOC 0 +#endif + +#if HEATSHRINK_DYNAMIC_ALLOC + /* Optional replacement of malloc/free */ + #define HEATSHRINK_MALLOC(SZ) malloc(SZ) + #define HEATSHRINK_FREE(P, SZ) free(P) +#else + /* Required parameters for static configuration */ + //#define HEATSHRINK_STATIC_INPUT_BUFFER_SIZE 32 + #define HEATSHRINK_STATIC_INPUT_BUFFER_SIZE 64 + //#define HEATSHRINK_STATIC_WINDOW_BITS 8 + #define HEATSHRINK_STATIC_WINDOW_BITS 9 + #define HEATSHRINK_STATIC_LOOKAHEAD_BITS 4 +#endif + +/* Turn on logging for debugging. */ +#define HEATSHRINK_DEBUGGING_LOGS 0 + +/* Use indexing for faster compression. (This requires additional space.) */ +#define HEATSHRINK_USE_INDEX 1 + +#endif diff --git a/src/microReticulum/Utilities/heatshrink/heatshrink_decoder.c b/src/microReticulum/Utilities/heatshrink/heatshrink_decoder.c new file mode 100644 index 0000000..0f118cf --- /dev/null +++ b/src/microReticulum/Utilities/heatshrink/heatshrink_decoder.c @@ -0,0 +1,367 @@ +#include +#include +#include "heatshrink_decoder.h" + +/* States for the polling state machine. */ +typedef enum { + HSDS_TAG_BIT, /* tag bit */ + HSDS_YIELD_LITERAL, /* ready to yield literal byte */ + HSDS_BACKREF_INDEX_MSB, /* most significant byte of index */ + HSDS_BACKREF_INDEX_LSB, /* least significant byte of index */ + HSDS_BACKREF_COUNT_MSB, /* most significant byte of count */ + HSDS_BACKREF_COUNT_LSB, /* least significant byte of count */ + HSDS_YIELD_BACKREF, /* ready to yield back-reference */ +} HSD_state; + +#if HEATSHRINK_DEBUGGING_LOGS +#include +#include +#include +#define LOG(...) fprintf(stderr, __VA_ARGS__) +#define ASSERT(X) assert(X) +static const char *state_names[] = { + "tag_bit", + "yield_literal", + "backref_index_msb", + "backref_index_lsb", + "backref_count_msb", + "backref_count_lsb", + "yield_backref", +}; +#else +#define LOG(...) /* no-op */ +#define ASSERT(X) /* no-op */ +#endif + +typedef struct { + uint8_t *buf; /* output buffer */ + size_t buf_size; /* buffer size */ + size_t *output_size; /* bytes pushed to buffer, so far */ +} output_info; + +#define NO_BITS ((uint16_t)-1) + +/* Forward references. */ +static uint16_t get_bits(heatshrink_decoder *hsd, uint8_t count); +static void push_byte(heatshrink_decoder *hsd, output_info *oi, uint8_t byte); + +#if HEATSHRINK_DYNAMIC_ALLOC +heatshrink_decoder *heatshrink_decoder_alloc(uint16_t input_buffer_size, + uint8_t window_sz2, + uint8_t lookahead_sz2) { + if ((window_sz2 < HEATSHRINK_MIN_WINDOW_BITS) || + (window_sz2 > HEATSHRINK_MAX_WINDOW_BITS) || + (input_buffer_size == 0) || + (lookahead_sz2 < HEATSHRINK_MIN_LOOKAHEAD_BITS) || + (lookahead_sz2 >= window_sz2)) { + return NULL; + } + size_t buffers_sz = (1 << window_sz2) + input_buffer_size; + size_t sz = sizeof(heatshrink_decoder) + buffers_sz; + heatshrink_decoder *hsd = HEATSHRINK_MALLOC(sz); + if (hsd == NULL) { return NULL; } + hsd->input_buffer_size = input_buffer_size; + hsd->window_sz2 = window_sz2; + hsd->lookahead_sz2 = lookahead_sz2; + heatshrink_decoder_reset(hsd); + LOG("-- allocated decoder with buffer size of %zu (%zu + %u + %u)\n", + sz, sizeof(heatshrink_decoder), (1 << window_sz2), input_buffer_size); + return hsd; +} + +void heatshrink_decoder_free(heatshrink_decoder *hsd) { + size_t buffers_sz = (1 << hsd->window_sz2) + hsd->input_buffer_size; + size_t sz = sizeof(heatshrink_decoder) + buffers_sz; + HEATSHRINK_FREE(hsd, sz); + (void)sz; /* may not be used by free */ +} +#endif + +void heatshrink_decoder_reset(heatshrink_decoder *hsd) { + size_t buf_sz = 1 << HEATSHRINK_DECODER_WINDOW_BITS(hsd); + size_t input_sz = HEATSHRINK_DECODER_INPUT_BUFFER_SIZE(hsd); + memset(hsd->buffers, 0, buf_sz + input_sz); + hsd->state = HSDS_TAG_BIT; + hsd->input_size = 0; + hsd->input_index = 0; + hsd->bit_index = 0x00; + hsd->current_byte = 0x00; + hsd->output_count = 0; + hsd->output_index = 0; + hsd->head_index = 0; +} + +/* Copy SIZE bytes into the decoder's input buffer, if it will fit. */ +HSD_sink_res heatshrink_decoder_sink(heatshrink_decoder *hsd, + uint8_t *in_buf, size_t size, size_t *input_size) { + if ((hsd == NULL) || (in_buf == NULL) || (input_size == NULL)) { + return HSDR_SINK_ERROR_NULL; + } + + size_t rem = HEATSHRINK_DECODER_INPUT_BUFFER_SIZE(hsd) - hsd->input_size; + if (rem == 0) { + *input_size = 0; + return HSDR_SINK_FULL; + } + + size = rem < size ? rem : size; + LOG("-- sinking %zd bytes\n", size); + /* copy into input buffer (at head of buffers) */ + memcpy(&hsd->buffers[hsd->input_size], in_buf, size); + hsd->input_size += size; + *input_size = size; + return HSDR_SINK_OK; +} + + +/***************** + * Decompression * + *****************/ + +#define BACKREF_COUNT_BITS(HSD) (HEATSHRINK_DECODER_LOOKAHEAD_BITS(HSD)) +#define BACKREF_INDEX_BITS(HSD) (HEATSHRINK_DECODER_WINDOW_BITS(HSD)) + +// States +static HSD_state st_tag_bit(heatshrink_decoder *hsd); +static HSD_state st_yield_literal(heatshrink_decoder *hsd, + output_info *oi); +static HSD_state st_backref_index_msb(heatshrink_decoder *hsd); +static HSD_state st_backref_index_lsb(heatshrink_decoder *hsd); +static HSD_state st_backref_count_msb(heatshrink_decoder *hsd); +static HSD_state st_backref_count_lsb(heatshrink_decoder *hsd); +static HSD_state st_yield_backref(heatshrink_decoder *hsd, + output_info *oi); + +HSD_poll_res heatshrink_decoder_poll(heatshrink_decoder *hsd, + uint8_t *out_buf, size_t out_buf_size, size_t *output_size) { + if ((hsd == NULL) || (out_buf == NULL) || (output_size == NULL)) { + return HSDR_POLL_ERROR_NULL; + } + *output_size = 0; + + output_info oi; + oi.buf = out_buf; + oi.buf_size = out_buf_size; + oi.output_size = output_size; + + while (1) { + LOG("-- poll, state is %d (%s), input_size %d\n", + hsd->state, state_names[hsd->state], hsd->input_size); + uint8_t in_state = hsd->state; + switch (in_state) { + case HSDS_TAG_BIT: + hsd->state = st_tag_bit(hsd); + break; + case HSDS_YIELD_LITERAL: + hsd->state = st_yield_literal(hsd, &oi); + break; + case HSDS_BACKREF_INDEX_MSB: + hsd->state = st_backref_index_msb(hsd); + break; + case HSDS_BACKREF_INDEX_LSB: + hsd->state = st_backref_index_lsb(hsd); + break; + case HSDS_BACKREF_COUNT_MSB: + hsd->state = st_backref_count_msb(hsd); + break; + case HSDS_BACKREF_COUNT_LSB: + hsd->state = st_backref_count_lsb(hsd); + break; + case HSDS_YIELD_BACKREF: + hsd->state = st_yield_backref(hsd, &oi); + break; + default: + return HSDR_POLL_ERROR_UNKNOWN; + } + + /* If the current state cannot advance, check if input or output + * buffer are exhausted. */ + if (hsd->state == in_state) { + if (*output_size == out_buf_size) { return HSDR_POLL_MORE; } + return HSDR_POLL_EMPTY; + } + } +} + +static HSD_state st_tag_bit(heatshrink_decoder *hsd) { + uint32_t bits = get_bits(hsd, 1); // get tag bit + if (bits == NO_BITS) { + return HSDS_TAG_BIT; + } else if (bits) { + return HSDS_YIELD_LITERAL; + } else if (HEATSHRINK_DECODER_WINDOW_BITS(hsd) > 8) { + return HSDS_BACKREF_INDEX_MSB; + } else { + hsd->output_index = 0; + return HSDS_BACKREF_INDEX_LSB; + } +} + +static HSD_state st_yield_literal(heatshrink_decoder *hsd, + output_info *oi) { + /* Emit a repeated section from the window buffer, and add it (again) + * to the window buffer. (Note that the repetition can include + * itself.)*/ + if (*oi->output_size < oi->buf_size) { + uint16_t byte = get_bits(hsd, 8); + if (byte == NO_BITS) { return HSDS_YIELD_LITERAL; } /* out of input */ + uint8_t *buf = &hsd->buffers[HEATSHRINK_DECODER_INPUT_BUFFER_SIZE(hsd)]; + uint16_t mask = (1 << HEATSHRINK_DECODER_WINDOW_BITS(hsd)) - 1; + uint8_t c = byte & 0xFF; + LOG("-- emitting literal byte 0x%02x ('%c')\n", c, isprint(c) ? c : '.'); + buf[hsd->head_index++ & mask] = c; + push_byte(hsd, oi, c); + return HSDS_TAG_BIT; + } else { + return HSDS_YIELD_LITERAL; + } +} + +static HSD_state st_backref_index_msb(heatshrink_decoder *hsd) { + uint8_t bit_ct = BACKREF_INDEX_BITS(hsd); + ASSERT(bit_ct > 8); + uint16_t bits = get_bits(hsd, bit_ct - 8); + LOG("-- backref index (msb), got 0x%04x (+1)\n", bits); + if (bits == NO_BITS) { return HSDS_BACKREF_INDEX_MSB; } + hsd->output_index = bits << 8; + return HSDS_BACKREF_INDEX_LSB; +} + +static HSD_state st_backref_index_lsb(heatshrink_decoder *hsd) { + uint8_t bit_ct = BACKREF_INDEX_BITS(hsd); + uint16_t bits = get_bits(hsd, bit_ct < 8 ? bit_ct : 8); + LOG("-- backref index (lsb), got 0x%04x (+1)\n", bits); + if (bits == NO_BITS) { return HSDS_BACKREF_INDEX_LSB; } + hsd->output_index |= bits; + hsd->output_index++; + uint8_t br_bit_ct = BACKREF_COUNT_BITS(hsd); + hsd->output_count = 0; + return (br_bit_ct > 8) ? HSDS_BACKREF_COUNT_MSB : HSDS_BACKREF_COUNT_LSB; +} + +static HSD_state st_backref_count_msb(heatshrink_decoder *hsd) { + uint8_t br_bit_ct = BACKREF_COUNT_BITS(hsd); + ASSERT(br_bit_ct > 8); + uint16_t bits = get_bits(hsd, br_bit_ct - 8); + LOG("-- backref count (msb), got 0x%04x (+1)\n", bits); + if (bits == NO_BITS) { return HSDS_BACKREF_COUNT_MSB; } + hsd->output_count = bits << 8; + return HSDS_BACKREF_COUNT_LSB; +} + +static HSD_state st_backref_count_lsb(heatshrink_decoder *hsd) { + uint8_t br_bit_ct = BACKREF_COUNT_BITS(hsd); + uint16_t bits = get_bits(hsd, br_bit_ct < 8 ? br_bit_ct : 8); + LOG("-- backref count (lsb), got 0x%04x (+1)\n", bits); + if (bits == NO_BITS) { return HSDS_BACKREF_COUNT_LSB; } + hsd->output_count |= bits; + hsd->output_count++; + return HSDS_YIELD_BACKREF; +} + +static HSD_state st_yield_backref(heatshrink_decoder *hsd, + output_info *oi) { + size_t count = oi->buf_size - *oi->output_size; + if (count > 0) { + size_t i = 0; + if (hsd->output_count < count) count = hsd->output_count; + uint8_t *buf = &hsd->buffers[HEATSHRINK_DECODER_INPUT_BUFFER_SIZE(hsd)]; + uint16_t mask = (1 << HEATSHRINK_DECODER_WINDOW_BITS(hsd)) - 1; + uint16_t neg_offset = hsd->output_index; + LOG("-- emitting %zu bytes from -%u bytes back\n", count, neg_offset); + ASSERT(neg_offset <= mask + 1); + ASSERT(count <= (size_t)(1 << BACKREF_COUNT_BITS(hsd))); + + for (i=0; ihead_index - neg_offset) & mask]; + push_byte(hsd, oi, c); + buf[hsd->head_index & mask] = c; + hsd->head_index++; + LOG(" -- ++ 0x%02x\n", c); + } + hsd->output_count -= count; + if (hsd->output_count == 0) { return HSDS_TAG_BIT; } + } + return HSDS_YIELD_BACKREF; +} + +/* Get the next COUNT bits from the input buffer, saving incremental progress. + * Returns NO_BITS on end of input, or if more than 15 bits are requested. */ +static uint16_t get_bits(heatshrink_decoder *hsd, uint8_t count) { + uint16_t accumulator = 0; + int i = 0; + if (count > 15) { return NO_BITS; } + LOG("-- popping %u bit(s)\n", count); + + /* If we aren't able to get COUNT bits, suspend immediately, because we + * don't track how many bits of COUNT we've accumulated before suspend. */ + if (hsd->input_size == 0) { + if (hsd->bit_index < (1 << (count - 1))) { return NO_BITS; } + } + + for (i = 0; i < count; i++) { + if (hsd->bit_index == 0x00) { + if (hsd->input_size == 0) { + LOG(" -- out of bits, suspending w/ accumulator of %u (0x%02x)\n", + accumulator, accumulator); + return NO_BITS; + } + hsd->current_byte = hsd->buffers[hsd->input_index++]; + LOG(" -- pulled byte 0x%02x\n", hsd->current_byte); + if (hsd->input_index == hsd->input_size) { + hsd->input_index = 0; /* input is exhausted */ + hsd->input_size = 0; + } + hsd->bit_index = 0x80; + } + accumulator <<= 1; + if (hsd->current_byte & hsd->bit_index) { + accumulator |= 0x01; + if (0) { + LOG(" -- got 1, accumulator 0x%04x, bit_index 0x%02x\n", + accumulator, hsd->bit_index); + } + } else { + if (0) { + LOG(" -- got 0, accumulator 0x%04x, bit_index 0x%02x\n", + accumulator, hsd->bit_index); + } + } + hsd->bit_index >>= 1; + } + + if (count > 1) { LOG(" -- accumulated %08x\n", accumulator); } + return accumulator; +} + +HSD_finish_res heatshrink_decoder_finish(heatshrink_decoder *hsd) { + if (hsd == NULL) { return HSDR_FINISH_ERROR_NULL; } + switch (hsd->state) { + case HSDS_TAG_BIT: + return hsd->input_size == 0 ? HSDR_FINISH_DONE : HSDR_FINISH_MORE; + + /* If we want to finish with no input, but are in these states, it's + * because the 0-bit padding to the last byte looks like a backref + * marker bit followed by all 0s for index and count bits. */ + case HSDS_BACKREF_INDEX_LSB: + case HSDS_BACKREF_INDEX_MSB: + case HSDS_BACKREF_COUNT_LSB: + case HSDS_BACKREF_COUNT_MSB: + return hsd->input_size == 0 ? HSDR_FINISH_DONE : HSDR_FINISH_MORE; + + /* If the output stream is padded with 0xFFs (possibly due to being in + * flash memory), also explicitly check the input size rather than + * uselessly returning MORE but yielding 0 bytes when polling. */ + case HSDS_YIELD_LITERAL: + return hsd->input_size == 0 ? HSDR_FINISH_DONE : HSDR_FINISH_MORE; + + default: + return HSDR_FINISH_MORE; + } +} + +static void push_byte(heatshrink_decoder *hsd, output_info *oi, uint8_t byte) { + LOG(" -- pushing byte: 0x%02x ('%c')\n", byte, isprint(byte) ? byte : '.'); + oi->buf[(*oi->output_size)++] = byte; + (void)hsd; +} diff --git a/src/microReticulum/Utilities/heatshrink/heatshrink_decoder.h b/src/microReticulum/Utilities/heatshrink/heatshrink_decoder.h new file mode 100644 index 0000000..bda8399 --- /dev/null +++ b/src/microReticulum/Utilities/heatshrink/heatshrink_decoder.h @@ -0,0 +1,100 @@ +#ifndef HEATSHRINK_DECODER_H +#define HEATSHRINK_DECODER_H + +#include +#include +#include "heatshrink_common.h" +#include "heatshrink_config.h" + +typedef enum { + HSDR_SINK_OK, /* data sunk, ready to poll */ + HSDR_SINK_FULL, /* out of space in internal buffer */ + HSDR_SINK_ERROR_NULL=-1, /* NULL argument */ +} HSD_sink_res; + +typedef enum { + HSDR_POLL_EMPTY, /* input exhausted */ + HSDR_POLL_MORE, /* more data remaining, call again w/ fresh output buffer */ + HSDR_POLL_ERROR_NULL=-1, /* NULL arguments */ + HSDR_POLL_ERROR_UNKNOWN=-2, +} HSD_poll_res; + +typedef enum { + HSDR_FINISH_DONE, /* output is done */ + HSDR_FINISH_MORE, /* more output remains */ + HSDR_FINISH_ERROR_NULL=-1, /* NULL arguments */ +} HSD_finish_res; + +#if HEATSHRINK_DYNAMIC_ALLOC +#define HEATSHRINK_DECODER_INPUT_BUFFER_SIZE(BUF) \ + ((BUF)->input_buffer_size) +#define HEATSHRINK_DECODER_WINDOW_BITS(BUF) \ + ((BUF)->window_sz2) +#define HEATSHRINK_DECODER_LOOKAHEAD_BITS(BUF) \ + ((BUF)->lookahead_sz2) +#else +#define HEATSHRINK_DECODER_INPUT_BUFFER_SIZE(_) \ + HEATSHRINK_STATIC_INPUT_BUFFER_SIZE +#define HEATSHRINK_DECODER_WINDOW_BITS(_) \ + (HEATSHRINK_STATIC_WINDOW_BITS) +#define HEATSHRINK_DECODER_LOOKAHEAD_BITS(BUF) \ + (HEATSHRINK_STATIC_LOOKAHEAD_BITS) +#endif + +typedef struct { + uint16_t input_size; /* bytes in input buffer */ + uint16_t input_index; /* offset to next unprocessed input byte */ + uint16_t output_count; /* how many bytes to output */ + uint16_t output_index; /* index for bytes to output */ + uint16_t head_index; /* head of window buffer */ + uint8_t state; /* current state machine node */ + uint8_t current_byte; /* current byte of input */ + uint8_t bit_index; /* current bit index */ + +#if HEATSHRINK_DYNAMIC_ALLOC + /* Fields that are only used if dynamically allocated. */ + uint8_t window_sz2; /* window buffer bits */ + uint8_t lookahead_sz2; /* lookahead bits */ + uint16_t input_buffer_size; /* input buffer size */ + + /* Input buffer, then expansion window buffer */ + uint8_t buffers[]; +#else + /* Input buffer, then expansion window buffer */ + uint8_t buffers[(1 << HEATSHRINK_DECODER_WINDOW_BITS(_)) + + HEATSHRINK_DECODER_INPUT_BUFFER_SIZE(_)]; +#endif +} heatshrink_decoder; + +#if HEATSHRINK_DYNAMIC_ALLOC +/* Allocate a decoder with an input buffer of INPUT_BUFFER_SIZE bytes, + * an expansion buffer size of 2^WINDOW_SZ2, and a lookahead + * size of 2^lookahead_sz2. (The window buffer and lookahead sizes + * must match the settings used when the data was compressed.) + * Returns NULL on error. */ +heatshrink_decoder *heatshrink_decoder_alloc(uint16_t input_buffer_size, + uint8_t expansion_buffer_sz2, uint8_t lookahead_sz2); + +/* Free a decoder. */ +void heatshrink_decoder_free(heatshrink_decoder *hsd); +#endif + +/* Reset a decoder. */ +void heatshrink_decoder_reset(heatshrink_decoder *hsd); + +/* Sink at most SIZE bytes from IN_BUF into the decoder. *INPUT_SIZE is set to + * indicate how many bytes were actually sunk (in case a buffer was filled). */ +HSD_sink_res heatshrink_decoder_sink(heatshrink_decoder *hsd, + uint8_t *in_buf, size_t size, size_t *input_size); + +/* Poll for output from the decoder, copying at most OUT_BUF_SIZE bytes into + * OUT_BUF (setting *OUTPUT_SIZE to the actual amount copied). */ +HSD_poll_res heatshrink_decoder_poll(heatshrink_decoder *hsd, + uint8_t *out_buf, size_t out_buf_size, size_t *output_size); + +/* Notify the dencoder that the input stream is finished. + * If the return value is HSDR_FINISH_MORE, there is still more output, so + * call heatshrink_decoder_poll and repeat. */ +HSD_finish_res heatshrink_decoder_finish(heatshrink_decoder *hsd); + +#endif diff --git a/src/microReticulum/Utilities/heatshrink/heatshrink_encoder.c b/src/microReticulum/Utilities/heatshrink/heatshrink_encoder.c new file mode 100644 index 0000000..edf4abe --- /dev/null +++ b/src/microReticulum/Utilities/heatshrink/heatshrink_encoder.c @@ -0,0 +1,604 @@ +#include +#include +#include +#include "heatshrink_encoder.h" + +typedef enum { + HSES_NOT_FULL, /* input buffer not full enough */ + HSES_FILLED, /* buffer is full */ + HSES_SEARCH, /* searching for patterns */ + HSES_YIELD_TAG_BIT, /* yield tag bit */ + HSES_YIELD_LITERAL, /* emit literal byte */ + HSES_YIELD_BR_INDEX, /* yielding backref index */ + HSES_YIELD_BR_LENGTH, /* yielding backref length */ + HSES_SAVE_BACKLOG, /* copying buffer to backlog */ + HSES_FLUSH_BITS, /* flush bit buffer */ + HSES_DONE, /* done */ +} HSE_state; + +#if HEATSHRINK_DEBUGGING_LOGS +#include +#include +#include +#define LOG(...) fprintf(stderr, __VA_ARGS__) +#define ASSERT(X) assert(X) +static const char *state_names[] = { + "not_full", + "filled", + "search", + "yield_tag_bit", + "yield_literal", + "yield_br_index", + "yield_br_length", + "save_backlog", + "flush_bits", + "done", +}; +#else +#define LOG(...) /* no-op */ +#define ASSERT(X) /* no-op */ +#endif + +// Encoder flags +enum { + FLAG_IS_FINISHING = 0x01, +}; + +typedef struct { + uint8_t *buf; /* output buffer */ + size_t buf_size; /* buffer size */ + size_t *output_size; /* bytes pushed to buffer, so far */ +} output_info; + +#define MATCH_NOT_FOUND ((uint16_t)-1) + +static uint16_t get_input_offset(heatshrink_encoder *hse); +static uint16_t get_input_buffer_size(heatshrink_encoder *hse); +static uint16_t get_lookahead_size(heatshrink_encoder *hse); +static void add_tag_bit(heatshrink_encoder *hse, output_info *oi, uint8_t tag); +static int can_take_byte(output_info *oi); +static int is_finishing(heatshrink_encoder *hse); +static void save_backlog(heatshrink_encoder *hse); + +/* Push COUNT (max 8) bits to the output buffer, which has room. */ +static void push_bits(heatshrink_encoder *hse, uint8_t count, uint8_t bits, + output_info *oi); +static uint8_t push_outgoing_bits(heatshrink_encoder *hse, output_info *oi); +static void push_literal_byte(heatshrink_encoder *hse, output_info *oi); + +#if HEATSHRINK_DYNAMIC_ALLOC +heatshrink_encoder *heatshrink_encoder_alloc(uint8_t window_sz2, + uint8_t lookahead_sz2) { + if ((window_sz2 < HEATSHRINK_MIN_WINDOW_BITS) || + (window_sz2 > HEATSHRINK_MAX_WINDOW_BITS) || + (lookahead_sz2 < HEATSHRINK_MIN_LOOKAHEAD_BITS) || + (lookahead_sz2 >= window_sz2)) { + return NULL; + } + + /* Note: 2 * the window size is used because the buffer needs to fit + * (1 << window_sz2) bytes for the current input, and an additional + * (1 << window_sz2) bytes for the previous buffer of input, which + * will be scanned for useful backreferences. */ + size_t buf_sz = (2 << window_sz2); + + heatshrink_encoder *hse = HEATSHRINK_MALLOC(sizeof(*hse) + buf_sz); + if (hse == NULL) { return NULL; } + hse->window_sz2 = window_sz2; + hse->lookahead_sz2 = lookahead_sz2; + heatshrink_encoder_reset(hse); + +#if HEATSHRINK_USE_INDEX + size_t index_sz = buf_sz*sizeof(uint16_t); + hse->search_index = HEATSHRINK_MALLOC(index_sz + sizeof(struct hs_index)); + if (hse->search_index == NULL) { + HEATSHRINK_FREE(hse, sizeof(*hse) + buf_sz); + return NULL; + } + hse->search_index->size = index_sz; +#endif + + LOG("-- allocated encoder with buffer size of %zu (%u byte input size)\n", + buf_sz, get_input_buffer_size(hse)); + return hse; +} + +void heatshrink_encoder_free(heatshrink_encoder *hse) { + size_t buf_sz = (2 << HEATSHRINK_ENCODER_WINDOW_BITS(hse)); +#if HEATSHRINK_USE_INDEX + size_t index_sz = sizeof(struct hs_index) + hse->search_index->size; + HEATSHRINK_FREE(hse->search_index, index_sz); + (void)index_sz; +#endif + HEATSHRINK_FREE(hse, sizeof(heatshrink_encoder) + buf_sz); + (void)buf_sz; +} +#endif + +void heatshrink_encoder_reset(heatshrink_encoder *hse) { + size_t buf_sz = (2 << HEATSHRINK_ENCODER_WINDOW_BITS(hse)); + memset(hse->buffer, 0, buf_sz); + hse->input_size = 0; + hse->state = HSES_NOT_FULL; + hse->match_scan_index = 0; + hse->flags = 0; + hse->bit_index = 0x80; + hse->current_byte = 0x00; + hse->match_length = 0; + + hse->outgoing_bits = 0x0000; + hse->outgoing_bits_count = 0; + + #ifdef LOOP_DETECT + hse->loop_detect = (uint32_t)-1; + #endif +} + +HSE_sink_res heatshrink_encoder_sink(heatshrink_encoder *hse, + uint8_t *in_buf, size_t size, size_t *input_size) { + if ((hse == NULL) || (in_buf == NULL) || (input_size == NULL)) { + return HSER_SINK_ERROR_NULL; + } + + /* Sinking more content after saying the content is done, tsk tsk */ + if (is_finishing(hse)) { return HSER_SINK_ERROR_MISUSE; } + + /* Sinking more content before processing is done */ + if (hse->state != HSES_NOT_FULL) { return HSER_SINK_ERROR_MISUSE; } + + uint16_t write_offset = get_input_offset(hse) + hse->input_size; + uint16_t ibs = get_input_buffer_size(hse); + uint16_t rem = ibs - hse->input_size; + uint16_t cp_sz = rem < size ? rem : size; + + memcpy(&hse->buffer[write_offset], in_buf, cp_sz); + *input_size = cp_sz; + hse->input_size += cp_sz; + + LOG("-- sunk %u bytes (of %zu) into encoder at %d, input buffer now has %u\n", + cp_sz, size, write_offset, hse->input_size); + if (cp_sz == rem) { + LOG("-- internal buffer is now full\n"); + hse->state = HSES_FILLED; + } + + return HSER_SINK_OK; +} + + +/*************** + * Compression * + ***************/ + +static uint16_t find_longest_match(heatshrink_encoder *hse, uint16_t start, + uint16_t end, const uint16_t maxlen, uint16_t *match_length); +static void do_indexing(heatshrink_encoder *hse); + +static HSE_state st_step_search(heatshrink_encoder *hse); +static HSE_state st_yield_tag_bit(heatshrink_encoder *hse, + output_info *oi); +static HSE_state st_yield_literal(heatshrink_encoder *hse, + output_info *oi); +static HSE_state st_yield_br_index(heatshrink_encoder *hse, + output_info *oi); +static HSE_state st_yield_br_length(heatshrink_encoder *hse, + output_info *oi); +static HSE_state st_save_backlog(heatshrink_encoder *hse); +static HSE_state st_flush_bit_buffer(heatshrink_encoder *hse, + output_info *oi); + +HSE_poll_res heatshrink_encoder_poll(heatshrink_encoder *hse, + uint8_t *out_buf, size_t out_buf_size, size_t *output_size) { + if ((hse == NULL) || (out_buf == NULL) || (output_size == NULL)) { + return HSER_POLL_ERROR_NULL; + } + if (out_buf_size == 0) { + LOG("-- MISUSE: output buffer size is 0\n"); + return HSER_POLL_ERROR_MISUSE; + } + *output_size = 0; + + output_info oi; + oi.buf = out_buf; + oi.buf_size = out_buf_size; + oi.output_size = output_size; + + while (1) { + LOG("-- polling, state %u (%s), flags 0x%02x\n", + hse->state, state_names[hse->state], hse->flags); + + uint8_t in_state = hse->state; + switch (in_state) { + case HSES_NOT_FULL: + return HSER_POLL_EMPTY; + case HSES_FILLED: + do_indexing(hse); + hse->state = HSES_SEARCH; + break; + case HSES_SEARCH: + hse->state = st_step_search(hse); + break; + case HSES_YIELD_TAG_BIT: + hse->state = st_yield_tag_bit(hse, &oi); + break; + case HSES_YIELD_LITERAL: + hse->state = st_yield_literal(hse, &oi); + break; + case HSES_YIELD_BR_INDEX: + hse->state = st_yield_br_index(hse, &oi); + break; + case HSES_YIELD_BR_LENGTH: + hse->state = st_yield_br_length(hse, &oi); + break; + case HSES_SAVE_BACKLOG: + hse->state = st_save_backlog(hse); + break; + case HSES_FLUSH_BITS: + hse->state = st_flush_bit_buffer(hse, &oi); + case HSES_DONE: + return HSER_POLL_EMPTY; + default: + LOG("-- bad state %s\n", state_names[hse->state]); + return HSER_POLL_ERROR_MISUSE; + } + + if (hse->state == in_state) { + /* Check if output buffer is exhausted. */ + if (*output_size == out_buf_size) return HSER_POLL_MORE; + } + } +} + +HSE_finish_res heatshrink_encoder_finish(heatshrink_encoder *hse) { + if (hse == NULL) { return HSER_FINISH_ERROR_NULL; } + LOG("-- setting is_finishing flag\n"); + hse->flags |= FLAG_IS_FINISHING; + if (hse->state == HSES_NOT_FULL) { hse->state = HSES_FILLED; } + return hse->state == HSES_DONE ? HSER_FINISH_DONE : HSER_FINISH_MORE; +} + +static HSE_state st_step_search(heatshrink_encoder *hse) { + uint16_t window_length = get_input_buffer_size(hse); + uint16_t lookahead_sz = get_lookahead_size(hse); + uint16_t msi = hse->match_scan_index; + LOG("## step_search, scan @ +%d (%d/%d), input size %d\n", + msi, hse->input_size + msi, 2*window_length, hse->input_size); + + bool fin = is_finishing(hse); + if (msi > hse->input_size - (fin ? 1 : lookahead_sz)) { + /* Current search buffer is exhausted, copy it into the + * backlog and await more input. */ + LOG("-- end of search @ %d\n", msi); + return fin ? HSES_FLUSH_BITS : HSES_SAVE_BACKLOG; + } + + uint16_t input_offset = get_input_offset(hse); + uint16_t end = input_offset + msi; + uint16_t start = end - window_length; + + uint16_t max_possible = lookahead_sz; + if (hse->input_size - msi < lookahead_sz) { + max_possible = hse->input_size - msi; + } + + uint16_t match_length = 0; + uint16_t match_pos = find_longest_match(hse, + start, end, max_possible, &match_length); + + if (match_pos == MATCH_NOT_FOUND) { + LOG("ss Match not found\n"); + hse->match_scan_index++; + hse->match_length = 0; + return HSES_YIELD_TAG_BIT; + } else { + LOG("ss Found match of %d bytes at %d\n", match_length, match_pos); + hse->match_pos = match_pos; + hse->match_length = match_length; + ASSERT(match_pos <= 1 << HEATSHRINK_ENCODER_WINDOW_BITS(hse) /*window_length*/); + + return HSES_YIELD_TAG_BIT; + } +} + +static HSE_state st_yield_tag_bit(heatshrink_encoder *hse, + output_info *oi) { + if (can_take_byte(oi)) { + if (hse->match_length == 0) { + add_tag_bit(hse, oi, HEATSHRINK_LITERAL_MARKER); + return HSES_YIELD_LITERAL; + } else { + add_tag_bit(hse, oi, HEATSHRINK_BACKREF_MARKER); + hse->outgoing_bits = hse->match_pos - 1; + hse->outgoing_bits_count = HEATSHRINK_ENCODER_WINDOW_BITS(hse); + return HSES_YIELD_BR_INDEX; + } + } else { + return HSES_YIELD_TAG_BIT; /* output is full, continue */ + } +} + +static HSE_state st_yield_literal(heatshrink_encoder *hse, + output_info *oi) { + if (can_take_byte(oi)) { + push_literal_byte(hse, oi); + return HSES_SEARCH; + } else { + return HSES_YIELD_LITERAL; + } +} + +static HSE_state st_yield_br_index(heatshrink_encoder *hse, + output_info *oi) { + if (can_take_byte(oi)) { + LOG("-- yielding backref index %u\n", hse->match_pos); + if (push_outgoing_bits(hse, oi) > 0) { + return HSES_YIELD_BR_INDEX; /* continue */ + } else { + hse->outgoing_bits = hse->match_length - 1; + hse->outgoing_bits_count = HEATSHRINK_ENCODER_LOOKAHEAD_BITS(hse); + return HSES_YIELD_BR_LENGTH; /* done */ + } + } else { + return HSES_YIELD_BR_INDEX; /* continue */ + } +} + +static HSE_state st_yield_br_length(heatshrink_encoder *hse, + output_info *oi) { + if (can_take_byte(oi)) { + LOG("-- yielding backref length %u\n", hse->match_length); + if (push_outgoing_bits(hse, oi) > 0) { + return HSES_YIELD_BR_LENGTH; + } else { + hse->match_scan_index += hse->match_length; + hse->match_length = 0; + return HSES_SEARCH; + } + } else { + return HSES_YIELD_BR_LENGTH; + } +} + +static HSE_state st_save_backlog(heatshrink_encoder *hse) { + LOG("-- saving backlog\n"); + save_backlog(hse); + return HSES_NOT_FULL; +} + +static HSE_state st_flush_bit_buffer(heatshrink_encoder *hse, + output_info *oi) { + if (hse->bit_index == 0x80) { + LOG("-- done!\n"); + return HSES_DONE; + } else if (can_take_byte(oi)) { + LOG("-- flushing remaining byte (bit_index == 0x%02x)\n", hse->bit_index); + oi->buf[(*oi->output_size)++] = hse->current_byte; + LOG("-- done!\n"); + return HSES_DONE; + } else { + return HSES_FLUSH_BITS; + } +} + +static void add_tag_bit(heatshrink_encoder *hse, output_info *oi, uint8_t tag) { + LOG("-- adding tag bit: %d\n", tag); + push_bits(hse, 1, tag, oi); +} + +static uint16_t get_input_offset(heatshrink_encoder *hse) { + return get_input_buffer_size(hse); +} + +static uint16_t get_input_buffer_size(heatshrink_encoder *hse) { + return (1 << HEATSHRINK_ENCODER_WINDOW_BITS(hse)); + (void)hse; +} + +static uint16_t get_lookahead_size(heatshrink_encoder *hse) { + return (1 << HEATSHRINK_ENCODER_LOOKAHEAD_BITS(hse)); + (void)hse; +} + +static void do_indexing(heatshrink_encoder *hse) { +#if HEATSHRINK_USE_INDEX + /* Build an index array I that contains flattened linked lists + * for the previous instances of every byte in the buffer. + * + * For example, if buf[200] == 'x', then index[200] will either + * be an offset i such that buf[i] == 'x', or a negative offset + * to indicate end-of-list. This significantly speeds up matching, + * while only using sizeof(uint16_t)*sizeof(buffer) bytes of RAM. + * + * Future optimization options: + * 1. Since any negative value represents end-of-list, the other + * 15 bits could be used to improve the index dynamically. + * + * 2. Likewise, the last lookahead_sz bytes of the index will + * not be usable, so temporary data could be stored there to + * dynamically improve the index. + * */ + struct hs_index *hsi = HEATSHRINK_ENCODER_INDEX(hse); + int16_t last[256]; + memset(last, 0xFF, sizeof(last)); + + uint8_t * const data = hse->buffer; + int16_t * const index = hsi->index; + + const uint16_t input_offset = get_input_offset(hse); + const uint16_t end = input_offset + hse->input_size; + + for (uint16_t i=0; iflags & FLAG_IS_FINISHING; +} + +static int can_take_byte(output_info *oi) { + return *oi->output_size < oi->buf_size; +} + +/* Return the longest match for the bytes at buf[end:end+maxlen] between + * buf[start] and buf[end-1]. If no match is found, return -1. */ +static uint16_t find_longest_match(heatshrink_encoder *hse, uint16_t start, + uint16_t end, const uint16_t maxlen, uint16_t *match_length) { + LOG("-- scanning for match of buf[%u:%u] between buf[%u:%u] (max %u bytes)\n", + end, end + maxlen, start, end + maxlen - 1, maxlen); + uint8_t *buf = hse->buffer; + + uint16_t match_maxlen = 0; + uint16_t match_index = MATCH_NOT_FOUND; + + uint16_t len = 0; + uint8_t * const needlepoint = &buf[end]; +#if HEATSHRINK_USE_INDEX + struct hs_index *hsi = HEATSHRINK_ENCODER_INDEX(hse); + int16_t pos = hsi->index[end]; + + while (pos - (int16_t)start >= 0) { + uint8_t * const pospoint = &buf[pos]; + len = 0; + + /* Only check matches that will potentially beat the current maxlen. + * This is redundant with the index if match_maxlen is 0, but the + * added branch overhead to check if it == 0 seems to be worse. */ + if (pospoint[match_maxlen] != needlepoint[match_maxlen]) { + pos = hsi->index[pos]; + continue; + } + + for (len = 1; len < maxlen; len++) { + if (pospoint[len] != needlepoint[len]) break; + } + + if (len > match_maxlen) { + match_maxlen = len; + match_index = pos; + if (len == maxlen) { break; } /* won't find better */ + } + pos = hsi->index[pos]; + } +#else + for (int16_t pos=end - 1; pos - (int16_t)start >= 0; pos--) { + uint8_t * const pospoint = &buf[pos]; + if ((pospoint[match_maxlen] == needlepoint[match_maxlen]) + && (*pospoint == *needlepoint)) { + for (len=1; len cmp buf[%d] == 0x%02x against %02x (start %u)\n", + pos + len, pospoint[len], needlepoint[len], start); + } + if (pospoint[len] != needlepoint[len]) { break; } + } + if (len > match_maxlen) { + match_maxlen = len; + match_index = pos; + if (len == maxlen) { break; } /* don't keep searching */ + } + } + } +#endif + + const size_t break_even_point = + (1 + HEATSHRINK_ENCODER_WINDOW_BITS(hse) + + HEATSHRINK_ENCODER_LOOKAHEAD_BITS(hse)); + + /* Instead of comparing break_even_point against 8*match_maxlen, + * compare match_maxlen against break_even_point/8 to avoid + * overflow. Since MIN_WINDOW_BITS and MIN_LOOKAHEAD_BITS are 4 and + * 3, respectively, break_even_point/8 will always be at least 1. */ + if (match_maxlen > (break_even_point / 8)) { + LOG("-- best match: %u bytes at -%u\n", + match_maxlen, end - match_index); + *match_length = match_maxlen; + return end - match_index; + } + LOG("-- none found\n"); + return MATCH_NOT_FOUND; +} + +static uint8_t push_outgoing_bits(heatshrink_encoder *hse, output_info *oi) { + uint8_t count = 0; + uint8_t bits = 0; + if (hse->outgoing_bits_count > 8) { + count = 8; + bits = hse->outgoing_bits >> (hse->outgoing_bits_count - 8); + } else { + count = hse->outgoing_bits_count; + bits = hse->outgoing_bits; + } + + if (count > 0) { + LOG("-- pushing %d outgoing bits: 0x%02x\n", count, bits); + push_bits(hse, count, bits, oi); + hse->outgoing_bits_count -= count; + } + return count; +} + +/* Push COUNT (max 8) bits to the output buffer, which has room. + * Bytes are set from the lowest bits, up. */ +static void push_bits(heatshrink_encoder *hse, uint8_t count, uint8_t bits, + output_info *oi) { + ASSERT(count <= 8); + LOG("++ push_bits: %d bits, input of 0x%02x\n", count, bits); + + /* If adding a whole byte and at the start of a new output byte, + * just push it through whole and skip the bit IO loop. */ + if (count == 8 && hse->bit_index == 0x80) { + oi->buf[(*oi->output_size)++] = bits; + } else { + for (int i=count - 1; i>=0; i--) { + bool bit = bits & (1 << i); + if (bit) { hse->current_byte |= hse->bit_index; } + if (0) { + LOG(" -- setting bit %d at bit index 0x%02x, byte => 0x%02x\n", + bit ? 1 : 0, hse->bit_index, hse->current_byte); + } + hse->bit_index >>= 1; + if (hse->bit_index == 0x00) { + hse->bit_index = 0x80; + LOG(" > pushing byte 0x%02x\n", hse->current_byte); + oi->buf[(*oi->output_size)++] = hse->current_byte; + hse->current_byte = 0x00; + } + } + } +} + +static void push_literal_byte(heatshrink_encoder *hse, output_info *oi) { + uint16_t processed_offset = hse->match_scan_index - 1; + uint16_t input_offset = get_input_offset(hse) + processed_offset; + uint8_t c = hse->buffer[input_offset]; + LOG("-- yielded literal byte 0x%02x ('%c') from +%d\n", + c, isprint(c) ? c : '.', input_offset); + push_bits(hse, 8, c, oi); +} + +static void save_backlog(heatshrink_encoder *hse) { + size_t input_buf_sz = get_input_buffer_size(hse); + + uint16_t msi = hse->match_scan_index; + + /* Copy processed data to beginning of buffer, so it can be + * used for future matches. Don't bother checking whether the + * input is less than the maximum size, because if it isn't, + * we're done anyway. */ + uint16_t rem = input_buf_sz - msi; // unprocessed bytes + uint16_t shift_sz = input_buf_sz + rem; + + memmove(&hse->buffer[0], + &hse->buffer[input_buf_sz - rem], + shift_sz); + + hse->match_scan_index = 0; + hse->input_size -= input_buf_sz - rem; +} diff --git a/src/microReticulum/Utilities/heatshrink/heatshrink_encoder.h b/src/microReticulum/Utilities/heatshrink/heatshrink_encoder.h new file mode 100644 index 0000000..18c1773 --- /dev/null +++ b/src/microReticulum/Utilities/heatshrink/heatshrink_encoder.h @@ -0,0 +1,109 @@ +#ifndef HEATSHRINK_ENCODER_H +#define HEATSHRINK_ENCODER_H + +#include +#include +#include "heatshrink_common.h" +#include "heatshrink_config.h" + +typedef enum { + HSER_SINK_OK, /* data sunk into input buffer */ + HSER_SINK_ERROR_NULL=-1, /* NULL argument */ + HSER_SINK_ERROR_MISUSE=-2, /* API misuse */ +} HSE_sink_res; + +typedef enum { + HSER_POLL_EMPTY, /* input exhausted */ + HSER_POLL_MORE, /* poll again for more output */ + HSER_POLL_ERROR_NULL=-1, /* NULL argument */ + HSER_POLL_ERROR_MISUSE=-2, /* API misuse */ +} HSE_poll_res; + +typedef enum { + HSER_FINISH_DONE, /* encoding is complete */ + HSER_FINISH_MORE, /* more output remaining; use poll */ + HSER_FINISH_ERROR_NULL=-1, /* NULL argument */ +} HSE_finish_res; + +#if HEATSHRINK_DYNAMIC_ALLOC +#define HEATSHRINK_ENCODER_WINDOW_BITS(HSE) \ + ((HSE)->window_sz2) +#define HEATSHRINK_ENCODER_LOOKAHEAD_BITS(HSE) \ + ((HSE)->lookahead_sz2) +#define HEATSHRINK_ENCODER_INDEX(HSE) \ + ((HSE)->search_index) +struct hs_index { + uint16_t size; + int16_t index[]; +}; +#else +#define HEATSHRINK_ENCODER_WINDOW_BITS(_) \ + (HEATSHRINK_STATIC_WINDOW_BITS) +#define HEATSHRINK_ENCODER_LOOKAHEAD_BITS(_) \ + (HEATSHRINK_STATIC_LOOKAHEAD_BITS) +#define HEATSHRINK_ENCODER_INDEX(HSE) \ + (&(HSE)->search_index) +struct hs_index { + uint16_t size; + int16_t index[2 << HEATSHRINK_STATIC_WINDOW_BITS]; +}; +#endif + +typedef struct { + uint16_t input_size; /* bytes in input buffer */ + uint16_t match_scan_index; + uint16_t match_length; + uint16_t match_pos; + uint16_t outgoing_bits; /* enqueued outgoing bits */ + uint8_t outgoing_bits_count; + uint8_t flags; + uint8_t state; /* current state machine node */ + uint8_t current_byte; /* current byte of output */ + uint8_t bit_index; /* current bit index */ +#if HEATSHRINK_DYNAMIC_ALLOC + uint8_t window_sz2; /* 2^n size of window */ + uint8_t lookahead_sz2; /* 2^n size of lookahead */ +#if HEATSHRINK_USE_INDEX + struct hs_index *search_index; +#endif + /* input buffer and / sliding window for expansion */ + uint8_t buffer[]; +#else + #if HEATSHRINK_USE_INDEX + struct hs_index search_index; + #endif + /* input buffer and / sliding window for expansion */ + uint8_t buffer[2 << HEATSHRINK_ENCODER_WINDOW_BITS(_)]; +#endif +} heatshrink_encoder; + +#if HEATSHRINK_DYNAMIC_ALLOC +/* Allocate a new encoder struct and its buffers. + * Returns NULL on error. */ +heatshrink_encoder *heatshrink_encoder_alloc(uint8_t window_sz2, + uint8_t lookahead_sz2); + +/* Free an encoder. */ +void heatshrink_encoder_free(heatshrink_encoder *hse); +#endif + +/* Reset an encoder. */ +void heatshrink_encoder_reset(heatshrink_encoder *hse); + +/* Sink up to SIZE bytes from IN_BUF into the encoder. + * INPUT_SIZE is set to the number of bytes actually sunk (in case a + * buffer was filled.). */ +HSE_sink_res heatshrink_encoder_sink(heatshrink_encoder *hse, + uint8_t *in_buf, size_t size, size_t *input_size); + +/* Poll for output from the encoder, copying at most OUT_BUF_SIZE bytes into + * OUT_BUF (setting *OUTPUT_SIZE to the actual amount copied). */ +HSE_poll_res heatshrink_encoder_poll(heatshrink_encoder *hse, + uint8_t *out_buf, size_t out_buf_size, size_t *output_size); + +/* Notify the encoder that the input stream is finished. + * If the return value is HSER_FINISH_MORE, there is still more output, so + * call heatshrink_encoder_poll and repeat. */ +HSE_finish_res heatshrink_encoder_finish(heatshrink_encoder *hse); + +#endif diff --git a/src/microReticulum/Utilities/tlsf.c b/src/microReticulum/Utilities/tlsf/tlsf.c similarity index 100% rename from src/microReticulum/Utilities/tlsf.c rename to src/microReticulum/Utilities/tlsf/tlsf.c diff --git a/src/microReticulum/Utilities/tlsf.h b/src/microReticulum/Utilities/tlsf/tlsf.h similarity index 100% rename from src/microReticulum/Utilities/tlsf.h rename to src/microReticulum/Utilities/tlsf/tlsf.h