From 18c276ce6a3c6cbfc86b3a7cbc7fa25a95da27de Mon Sep 17 00:00:00 2001 From: Alexander Lednev <57529355+iceseer@users.noreply.github.com> Date: Fri, 18 Apr 2025 12:08:45 +0300 Subject: [PATCH 1/9] [fix] cmake v.4.x support (#2439) Signed-off-by: iceseer --- cmake/Hunter/config.cmake | 74 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 74 insertions(+) diff --git a/cmake/Hunter/config.cmake b/cmake/Hunter/config.cmake index bac8806512..e79504fb64 100644 --- a/cmake/Hunter/config.cmake +++ b/cmake/Hunter/config.cmake @@ -22,6 +22,7 @@ hunter_config( URL https://github.com/bombela/backward-cpp/archive/refs/tags/v1.6.zip SHA1 93c4c843fc9308e62ac462459077d87dc6dd9885 CMAKE_ARGS BACKWARD_TESTS=OFF + CMAKE_POLICY_VERSION_MINIMUM=3.5 KEEP_PACKAGE_SOURCES ) @@ -82,6 +83,7 @@ if ("${WASM_COMPILER}" STREQUAL "WAVM") VERSION 1.0.14 CMAKE_ARGS WAVM_CXX_FLAGS=${WAVM_CXX_FLAGS} + CMAKE_POLICY_VERSION_MINIMUM=3.5 KEEP_PACKAGE_SOURCES ) endif () @@ -112,3 +114,75 @@ hunter_config( KEEP_PACKAGE_SOURCES ) + +hunter_config( + binaryen + VERSION 1.38.28-patch.3 + CMAKE_ARGS + CMAKE_POLICY_VERSION_MINIMUM=3.5 + KEEP_PACKAGE_SOURCES +) + +hunter_config( + ZLIB + VERSION 1.2.11-p1 + CMAKE_ARGS + CMAKE_POLICY_VERSION_MINIMUM=3.5 + KEEP_PACKAGE_SOURCES +) + +hunter_config( + Protobuf + VERSION 3.19.4-p0 + CMAKE_ARGS + CMAKE_POLICY_VERSION_MINIMUM=3.5 + KEEP_PACKAGE_SOURCES +) + +hunter_config( + c-ares + VERSION 1.14.0-p0 + CMAKE_ARGS + CMAKE_POLICY_VERSION_MINIMUM=3.5 + KEEP_PACKAGE_SOURCES +) + +hunter_config( + yaml-cpp + VERSION 0.6.2-p0 + CMAKE_ARGS + CMAKE_POLICY_VERSION_MINIMUM=3.5 + KEEP_PACKAGE_SOURCES +) + +hunter_config( + RapidJSON + VERSION 1.1.0-66eb606-p0 + CMAKE_ARGS + CMAKE_POLICY_VERSION_MINIMUM=3.5 + KEEP_PACKAGE_SOURCES +) + +hunter_config( + jsonrpc-lean + VERSION 0.0.0-6c093da8 + CMAKE_ARGS + CMAKE_POLICY_VERSION_MINIMUM=3.5 + KEEP_PACKAGE_SOURCES +) + +hunter_config( + Boost.DI + VERSION 1.1.0 + CMAKE_ARGS + CMAKE_POLICY_VERSION_MINIMUM=3.5 + KEEP_PACKAGE_SOURCES +) + +hunter_config( + zstd + VERSION 1.4.5-d73e2fb-p0 + CMAKE_ARGS + CMAKE_POLICY_VERSION_MINIMUM=3.5 + KEEP_PACKAGE_SOURCES +) From 42ecaba5ace0af8913211ccd3eb126073bfe36cc Mon Sep 17 00:00:00 2001 From: Ruslan Tushov Date: Fri, 18 Apr 2025 15:46:04 +0500 Subject: [PATCH 2/9] fix roles (#2440) Signed-off-by: turuslan --- core/network/types/roles.hpp | 80 +++++++++++++++++------------------- 1 file changed, 38 insertions(+), 42 deletions(-) diff --git a/core/network/types/roles.hpp b/core/network/types/roles.hpp index f58799324b..7018c594de 100644 --- a/core/network/types/roles.hpp +++ b/core/network/types/roles.hpp @@ -6,65 +6,61 @@ #pragma once -#include "scale/tie.hpp" +#include +#include + +#include "scale/kagome_scale.hpp" -// NOLINTBEGIN(cppcoreguidelines-pro-type-union-access) namespace kagome::network { struct Roles { - SCALE_TIE_ONLY(value); - - union { - struct { - /** - * Full node, does not participate in consensus. - */ - uint8_t full : 1; - - /** - * Light client node. - */ - uint8_t light : 1; + /// Full node, does not participate in consensus. + static constexpr uint8_t Full = 0b0000'0001; + /// Light client node. + static constexpr uint8_t Light = 0b0000'0010; + /// Act as an authority + static constexpr uint8_t Authority = 0b0000'0100; - /** - * Act as an authority - */ - uint8_t authority : 1; - - } flags; - uint8_t value; - }; - - Roles() : value(0) {} - Roles(uint8_t v) : value(v) {} + Roles() = default; + Roles(uint8_t value) : value_(value) {} // https://github.com/paritytech/polkadot-sdk/blob/6c3219ebe9231a0305f53c7b33cb558d46058062/substrate/client/network/common/src/role.rs#L101 bool isFull() const { - return flags.full != 0 or isAuthority(); + return (value_ & (Full | Authority)) != 0; } bool isAuthority() const { - return flags.authority != 0; + return (value_ & Authority) != 0; } // https://github.com/paritytech/polkadot-sdk/blob/6c3219ebe9231a0305f53c7b33cb558d46058062/substrate/client/network/common/src/role.rs#L111 bool isLight() const { return not isFull(); } - }; - inline std::string to_string(Roles r) { - switch (r.value) { - case 0: - return "none"; - case 1: - return "full"; - case 2: - return "light"; - case 4: - return "authority"; + uint8_t value() const { + return value_; } - return to_string(r.value); - } + + SCALE_CUSTOM_DECOMPOSITION(Roles, value_); + + friend std::string to_string(Roles roles) { + switch (roles.value_) { + case 0: + return "none"; + case 1: + return "full"; + case 2: + return "light"; + case 4: + return "authority"; + default: + return std::to_string(roles.value_); + } + } + + private: + uint8_t value_{0}; + }; + } // namespace kagome::network -// NOLINTEND(cppcoreguidelines-pro-type-union-access) From 29fe83edc811f63783a28d16a3097e9863583f32 Mon Sep 17 00:00:00 2001 From: Ruslan Tushov Date: Thu, 24 Apr 2025 15:05:58 +0500 Subject: [PATCH 3/9] remove view event check (#2445) - kNewHeads is called once for each block, so event is never duplicate - data race, 2 blocks became head before first event, so heads didn't change between first and second event, so second event was ignored Signed-off-by: turuslan (cherry picked from commit 76e011d575a52f2ac56e477e349b3f1c78edf248) --- core/network/impl/peer_view.cpp | 3 --- core/network/impl/protocols/parachain.cpp | 4 ++++ core/network/impl/protocols/parachain.hpp | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/core/network/impl/peer_view.cpp b/core/network/impl/peer_view.cpp index 58ee637abf..de34815758 100644 --- a/core/network/impl/peer_view.cpp +++ b/core/network/impl/peer_view.cpp @@ -92,9 +92,6 @@ namespace kagome::network { .new_head = header, .lost = {}, }; - if (event.view == my_view_) { - return; - } my_view_stripped_ = std::move(stripped_view); for (const auto &head : my_view_.heads_) { if (not event.view.contains(head)) { diff --git a/core/network/impl/protocols/parachain.cpp b/core/network/impl/protocols/parachain.cpp index 0623a931fa..12f361aa35 100644 --- a/core/network/impl/protocols/parachain.cpp +++ b/core/network/impl/protocols/parachain.cpp @@ -116,6 +116,10 @@ namespace kagome::network { } void ParachainProtocol::write(const View &view) { + if (view == last_sent_view_) { + return; + } + last_sent_view_ = view; auto message = encodeView(view); notifications_->peersOut([&](const PeerId &peer_id, size_t protocol_group) { notifications_->write(peer_id, protocol_group, message); diff --git a/core/network/impl/protocols/parachain.hpp b/core/network/impl/protocols/parachain.hpp index 8b4a112134..5015a41888 100644 --- a/core/network/impl/protocols/parachain.hpp +++ b/core/network/impl/protocols/parachain.hpp @@ -23,7 +23,6 @@ namespace kagome::network { class PeerView; struct Seconded; class ValidationObserver; - struct View; } // namespace kagome::network namespace kagome::network { @@ -76,6 +75,7 @@ namespace kagome::network { primitives::events::SyncStateSubscriptionEnginePtr sync_engine_; std::shared_ptr sync_sub_; std::shared_ptr my_view_sub_; + View last_sent_view_; // NOLINTEND(cppcoreguidelines-non-private-member-variables-in-classes) }; From 793596d555762aa1492d1a41c87e72a39d858b03 Mon Sep 17 00:00:00 2001 From: kamilsa Date: Tue, 29 Apr 2025 12:28:41 +0500 Subject: [PATCH 4/9] Update forked zlibe for mac (#2444) * Update to hunter with updated default zlib (cherry picked from commit 3f8b2c0059f88207ef9a3a4d7b945fb9dd4c691d) --- cmake/Hunter/config.cmake | 12 ++---------- cmake/Hunter/hunter-gate-url.cmake | 6 +++--- core/benchmark/block_execution_benchmark.cpp | 4 ++-- 3 files changed, 7 insertions(+), 15 deletions(-) diff --git a/cmake/Hunter/config.cmake b/cmake/Hunter/config.cmake index e79504fb64..1b02d99ffd 100644 --- a/cmake/Hunter/config.cmake +++ b/cmake/Hunter/config.cmake @@ -103,8 +103,8 @@ hunter_config( hunter_config( libp2p - URL https://github.com/libp2p/cpp-libp2p/archive/refs/tags/v0.1.34.zip - SHA1 ad725b991c6845d0e5d9f42c639ea62fa05593ff + VERSION 0.1.36 + KEEP_PACKAGE_SOURCES ) hunter_config( @@ -123,14 +123,6 @@ hunter_config( KEEP_PACKAGE_SOURCES ) -hunter_config( - ZLIB - VERSION 1.2.11-p1 - CMAKE_ARGS - CMAKE_POLICY_VERSION_MINIMUM=3.5 - KEEP_PACKAGE_SOURCES -) - hunter_config( Protobuf VERSION 3.19.4-p0 diff --git a/cmake/Hunter/hunter-gate-url.cmake b/cmake/Hunter/hunter-gate-url.cmake index f108408443..cce703ef86 100644 --- a/cmake/Hunter/hunter-gate-url.cmake +++ b/cmake/Hunter/hunter-gate-url.cmake @@ -1,5 +1,5 @@ HunterGate( - URL https://github.com/qdrvm/hunter/archive/refs/tags/v0.25.3-qdrvm31.zip - SHA1 f2d79db34e0adfb09acc010afbb7ca7aeefdbc01 - LOCAL + URL https://github.com/qdrvm/hunter/archive/refs/tags/v0.25.3-qdrvm37.tar.gz + SHA1 c02ff270e65c4f1b347c85b376eb344158a51ffa + LOCAL ) diff --git a/core/benchmark/block_execution_benchmark.cpp b/core/benchmark/block_execution_benchmark.cpp index 30e8fd275b..05f32696b9 100644 --- a/core/benchmark/block_execution_benchmark.cpp +++ b/core/benchmark/block_execution_benchmark.cpp @@ -9,7 +9,7 @@ #include #include -#include +#include #include "blockchain/block_tree.hpp" #include "primitives/runtime_dispatch_info.hpp" @@ -319,7 +319,7 @@ namespace kagome::benchmark { SL_VERBOSE(logger_, "Block #{}, {} ns", blocks[block_i].header.number, - duration_ns); + duration_ns.count()); } duration_stat_it++; } From 6b8a2d45e866e8de316e7dc9fc53d5932bcf349a Mon Sep 17 00:00:00 2001 From: Ruslan Tushov Date: Wed, 7 May 2025 19:06:14 +0500 Subject: [PATCH 5/9] pvf empty free workers (#2450) Signed-off-by: turuslan Co-authored-by: kamilsa (cherry picked from commit 90cc0dc38d5abd99933598d3c50d06fab3277ce2) --- core/parachain/pvf/workers.cpp | 28 ++++++++++++++++++---------- core/parachain/pvf/workers.hpp | 9 +++++---- 2 files changed, 23 insertions(+), 14 deletions(-) diff --git a/core/parachain/pvf/workers.cpp b/core/parachain/pvf/workers.cpp index 23c4d7981e..ff8e9fd02f 100644 --- a/core/parachain/pvf/workers.cpp +++ b/core/parachain/pvf/workers.cpp @@ -139,7 +139,8 @@ namespace kagome::parachain { void PvfWorkers::execute(Job &&job) { REINVOKE(*main_pool_handler_, execute, std::move(job)); - if (free_.empty()) { + auto free = findFree(job); + if (not free.has_value()) { if (used_ >= max_) { auto &queue = queues_[job.kind]; queue.emplace_back(std::move(job)); @@ -193,20 +194,25 @@ namespace kagome::parachain { }); return; } - findFree(std::move(job)); + runJob(free.value(), std::move(job)); } - void PvfWorkers::findFree(Job &&job) { - std::unique_lock lock(free_mutex_); + auto PvfWorkers::findFree(const Job &job) -> std::optional { auto it = std::ranges::find_if(free_, [&](const Worker &worker) { return worker.code_params == job.code_params; }); if (it == free_.end()) { it = free_.begin(); } - auto worker = *it; - free_.erase(it); - lock.unlock(); + if (it == free_.end()) { + return std::nullopt; + } + return it; + } + + void PvfWorkers::runJob(Free::iterator free_it, Job &&job) { + auto worker = *free_it; + free_.erase(free_it); writeCode(std::move(job), std::move(worker), std::make_shared(*this)); } @@ -254,9 +260,7 @@ namespace kagome::parachain { if (not r) { return; } - std::unique_lock lock(self->free_mutex_); self->free_.emplace_back(std::move(worker)); - lock.unlock(); self->dequeue(); }); auto cb = [cb_shared, timeout](outcome::result r) mutable { @@ -283,10 +287,14 @@ namespace kagome::parachain { if (queue.empty()) { continue; } + auto free = findFree(queue.front()); + if (not free.has_value()) { + break; + } auto job = std::move(queue.front()); queue.pop_front(); metric_queue_size_.at(kind)->set(queue.size()); - findFree(std::move(job)); + runJob(free.value(), std::move(job)); } } } // namespace kagome::parachain diff --git a/core/parachain/pvf/workers.hpp b/core/parachain/pvf/workers.hpp index 9612ce61b3..f84d10d593 100644 --- a/core/parachain/pvf/workers.hpp +++ b/core/parachain/pvf/workers.hpp @@ -9,7 +9,6 @@ #include #include #include -#include #include "metrics/metrics.hpp" #include "parachain/pvf/pvf_worker_types.hpp" @@ -71,7 +70,10 @@ namespace kagome::parachain { std::weak_ptr weak_self; }; - void findFree(Job &&job); + using Free = std::list; + + std::optional findFree(const Job &job); + void runJob(Free::iterator free_it, Job &&job); void writeCode(Job &&job, Worker &&worker, std::shared_ptr &&used); void call(Job &&job, Worker &&worker, std::shared_ptr &&used); void dequeue(); @@ -82,8 +84,7 @@ namespace kagome::parachain { std::filesystem::path exe_; size_t max_; PvfWorkerInputConfig worker_config_; - std::list free_; - mutable std::mutex free_mutex_; // Mutex for protecting free_ list + Free free_; size_t used_ = 0; std::unordered_map> queues_; From 63bad0aa4a05a19a98b7e4ed6311a2b61890b2c9 Mon Sep 17 00:00:00 2001 From: kamilsa Date: Tue, 10 Jun 2025 18:03:26 +0500 Subject: [PATCH 6/9] [fix] add garbage collection timer to peer manager (#2464) (cherry picked from commit 7b05d0f3b6ea471d30ae7217cd5ddd7e7696e685) --- core/network/impl/peer_manager_impl.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/core/network/impl/peer_manager_impl.cpp b/core/network/impl/peer_manager_impl.cpp index c86cc0b16a..f38bbb01fa 100644 --- a/core/network/impl/peer_manager_impl.cpp +++ b/core/network/impl/peer_manager_impl.cpp @@ -224,6 +224,9 @@ namespace kagome::network { // Do first alignment of peers count align(); + // Start timer for periodic collecting garbage + collectGarbage(); + return true; } From b9da9bf6ab95aaae31ccd72bd4ed8273a7f7ff19 Mon Sep 17 00:00:00 2001 From: kamilsa Date: Tue, 10 Jun 2025 19:08:03 +0500 Subject: [PATCH 7/9] Revert "Update forked zlibe for mac (#2444)" This reverts commit 793596d555762aa1492d1a41c87e72a39d858b03. --- cmake/Hunter/config.cmake | 12 ++++++++++-- cmake/Hunter/hunter-gate-url.cmake | 6 +++--- core/benchmark/block_execution_benchmark.cpp | 4 ++-- 3 files changed, 15 insertions(+), 7 deletions(-) diff --git a/cmake/Hunter/config.cmake b/cmake/Hunter/config.cmake index 1b02d99ffd..e79504fb64 100644 --- a/cmake/Hunter/config.cmake +++ b/cmake/Hunter/config.cmake @@ -103,8 +103,8 @@ hunter_config( hunter_config( libp2p - VERSION 0.1.36 - KEEP_PACKAGE_SOURCES + URL https://github.com/libp2p/cpp-libp2p/archive/refs/tags/v0.1.34.zip + SHA1 ad725b991c6845d0e5d9f42c639ea62fa05593ff ) hunter_config( @@ -123,6 +123,14 @@ hunter_config( KEEP_PACKAGE_SOURCES ) +hunter_config( + ZLIB + VERSION 1.2.11-p1 + CMAKE_ARGS + CMAKE_POLICY_VERSION_MINIMUM=3.5 + KEEP_PACKAGE_SOURCES +) + hunter_config( Protobuf VERSION 3.19.4-p0 diff --git a/cmake/Hunter/hunter-gate-url.cmake b/cmake/Hunter/hunter-gate-url.cmake index cce703ef86..f108408443 100644 --- a/cmake/Hunter/hunter-gate-url.cmake +++ b/cmake/Hunter/hunter-gate-url.cmake @@ -1,5 +1,5 @@ HunterGate( - URL https://github.com/qdrvm/hunter/archive/refs/tags/v0.25.3-qdrvm37.tar.gz - SHA1 c02ff270e65c4f1b347c85b376eb344158a51ffa - LOCAL + URL https://github.com/qdrvm/hunter/archive/refs/tags/v0.25.3-qdrvm31.zip + SHA1 f2d79db34e0adfb09acc010afbb7ca7aeefdbc01 + LOCAL ) diff --git a/core/benchmark/block_execution_benchmark.cpp b/core/benchmark/block_execution_benchmark.cpp index 05f32696b9..30e8fd275b 100644 --- a/core/benchmark/block_execution_benchmark.cpp +++ b/core/benchmark/block_execution_benchmark.cpp @@ -9,7 +9,7 @@ #include #include -#include +#include #include "blockchain/block_tree.hpp" #include "primitives/runtime_dispatch_info.hpp" @@ -319,7 +319,7 @@ namespace kagome::benchmark { SL_VERBOSE(logger_, "Block #{}, {} ns", blocks[block_i].header.number, - duration_ns.count()); + duration_ns); } duration_stat_it++; } From 6db13724954e552eeb0962c865bf988d4726750e Mon Sep 17 00:00:00 2001 From: kamilsa Date: Tue, 10 Jun 2025 19:08:03 +0500 Subject: [PATCH 8/9] Revert "fix roles (#2440)" This reverts commit 42ecaba5ace0af8913211ccd3eb126073bfe36cc. --- core/network/types/roles.hpp | 80 +++++++++++++++++++----------------- 1 file changed, 42 insertions(+), 38 deletions(-) diff --git a/core/network/types/roles.hpp b/core/network/types/roles.hpp index 7018c594de..f58799324b 100644 --- a/core/network/types/roles.hpp +++ b/core/network/types/roles.hpp @@ -6,61 +6,65 @@ #pragma once -#include -#include - -#include "scale/kagome_scale.hpp" +#include "scale/tie.hpp" +// NOLINTBEGIN(cppcoreguidelines-pro-type-union-access) namespace kagome::network { struct Roles { - /// Full node, does not participate in consensus. - static constexpr uint8_t Full = 0b0000'0001; - /// Light client node. - static constexpr uint8_t Light = 0b0000'0010; - /// Act as an authority - static constexpr uint8_t Authority = 0b0000'0100; + SCALE_TIE_ONLY(value); + + union { + struct { + /** + * Full node, does not participate in consensus. + */ + uint8_t full : 1; + + /** + * Light client node. + */ + uint8_t light : 1; - Roles() = default; - Roles(uint8_t value) : value_(value) {} + /** + * Act as an authority + */ + uint8_t authority : 1; + + } flags; + uint8_t value; + }; + + Roles() : value(0) {} + Roles(uint8_t v) : value(v) {} // https://github.com/paritytech/polkadot-sdk/blob/6c3219ebe9231a0305f53c7b33cb558d46058062/substrate/client/network/common/src/role.rs#L101 bool isFull() const { - return (value_ & (Full | Authority)) != 0; + return flags.full != 0 or isAuthority(); } bool isAuthority() const { - return (value_ & Authority) != 0; + return flags.authority != 0; } // https://github.com/paritytech/polkadot-sdk/blob/6c3219ebe9231a0305f53c7b33cb558d46058062/substrate/client/network/common/src/role.rs#L111 bool isLight() const { return not isFull(); } - - uint8_t value() const { - return value_; - } - - SCALE_CUSTOM_DECOMPOSITION(Roles, value_); - - friend std::string to_string(Roles roles) { - switch (roles.value_) { - case 0: - return "none"; - case 1: - return "full"; - case 2: - return "light"; - case 4: - return "authority"; - default: - return std::to_string(roles.value_); - } - } - - private: - uint8_t value_{0}; }; + inline std::string to_string(Roles r) { + switch (r.value) { + case 0: + return "none"; + case 1: + return "full"; + case 2: + return "light"; + case 4: + return "authority"; + } + return to_string(r.value); + } } // namespace kagome::network +// NOLINTEND(cppcoreguidelines-pro-type-union-access) From c4e9711754f2e4c2e85b2d894a4f4c17f7bf8104 Mon Sep 17 00:00:00 2001 From: kamilsa Date: Fri, 13 Jun 2025 14:38:17 +0500 Subject: [PATCH 9/9] [update] bump libp2p to version 0.1.37 --- cmake/Hunter/config.cmake | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmake/Hunter/config.cmake b/cmake/Hunter/config.cmake index e79504fb64..53620cf6c6 100644 --- a/cmake/Hunter/config.cmake +++ b/cmake/Hunter/config.cmake @@ -103,8 +103,8 @@ hunter_config( hunter_config( libp2p - URL https://github.com/libp2p/cpp-libp2p/archive/refs/tags/v0.1.34.zip - SHA1 ad725b991c6845d0e5d9f42c639ea62fa05593ff + URL https://github.com/libp2p/cpp-libp2p/archive/refs/tags/v0.1.37.zip + SHA1 0387feba109f0cd9c27e032a866831522a4f3f10 ) hunter_config(